braintrust 0.3.14__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- braintrust/__init__.py +4 -0
- braintrust/_generated_types.py +1200 -611
- braintrust/audit.py +2 -2
- braintrust/cli/eval.py +6 -7
- braintrust/cli/push.py +11 -11
- braintrust/conftest.py +1 -0
- braintrust/context.py +12 -17
- braintrust/contrib/temporal/__init__.py +16 -27
- braintrust/contrib/temporal/test_temporal.py +8 -3
- braintrust/devserver/auth.py +8 -8
- braintrust/devserver/cache.py +3 -4
- braintrust/devserver/cors.py +8 -7
- braintrust/devserver/dataset.py +3 -5
- braintrust/devserver/eval_hooks.py +7 -6
- braintrust/devserver/schemas.py +22 -19
- braintrust/devserver/server.py +19 -12
- braintrust/devserver/test_cached_login.py +4 -4
- braintrust/framework.py +128 -140
- braintrust/framework2.py +88 -87
- braintrust/functions/invoke.py +93 -53
- braintrust/functions/stream.py +3 -2
- braintrust/generated_types.py +17 -1
- braintrust/git_fields.py +11 -11
- braintrust/gitutil.py +2 -3
- braintrust/graph_util.py +10 -10
- braintrust/id_gen.py +2 -2
- braintrust/logger.py +346 -357
- braintrust/merge_row_batch.py +10 -9
- braintrust/oai.py +107 -24
- braintrust/otel/__init__.py +49 -49
- braintrust/otel/context.py +16 -30
- braintrust/otel/test_distributed_tracing.py +14 -11
- braintrust/otel/test_otel_bt_integration.py +32 -31
- braintrust/parameters.py +8 -8
- braintrust/prompt.py +14 -14
- braintrust/prompt_cache/disk_cache.py +5 -4
- braintrust/prompt_cache/lru_cache.py +3 -2
- braintrust/prompt_cache/prompt_cache.py +13 -14
- braintrust/queue.py +4 -4
- braintrust/score.py +4 -4
- braintrust/serializable_data_class.py +4 -4
- braintrust/span_identifier_v1.py +1 -2
- braintrust/span_identifier_v2.py +3 -4
- braintrust/span_identifier_v3.py +23 -20
- braintrust/span_identifier_v4.py +34 -25
- braintrust/test_framework.py +16 -6
- braintrust/test_helpers.py +5 -5
- braintrust/test_id_gen.py +2 -3
- braintrust/test_otel.py +61 -53
- braintrust/test_queue.py +0 -1
- braintrust/test_score.py +1 -3
- braintrust/test_span_components.py +29 -44
- braintrust/util.py +9 -8
- braintrust/version.py +2 -2
- braintrust/wrappers/_anthropic_utils.py +4 -4
- braintrust/wrappers/agno/__init__.py +3 -4
- braintrust/wrappers/agno/agent.py +1 -2
- braintrust/wrappers/agno/function_call.py +1 -2
- braintrust/wrappers/agno/model.py +1 -2
- braintrust/wrappers/agno/team.py +1 -2
- braintrust/wrappers/agno/utils.py +12 -12
- braintrust/wrappers/anthropic.py +7 -8
- braintrust/wrappers/claude_agent_sdk/__init__.py +3 -4
- braintrust/wrappers/claude_agent_sdk/_wrapper.py +29 -27
- braintrust/wrappers/dspy.py +15 -17
- braintrust/wrappers/google_genai/__init__.py +16 -16
- braintrust/wrappers/langchain.py +22 -24
- braintrust/wrappers/litellm.py +4 -3
- braintrust/wrappers/openai.py +15 -15
- braintrust/wrappers/pydantic_ai.py +1204 -0
- braintrust/wrappers/test_agno.py +0 -1
- braintrust/wrappers/test_dspy.py +0 -1
- braintrust/wrappers/test_google_genai.py +2 -3
- braintrust/wrappers/test_litellm.py +0 -1
- braintrust/wrappers/test_oai_attachments.py +322 -0
- braintrust/wrappers/test_pydantic_ai_integration.py +1788 -0
- braintrust/wrappers/{test_pydantic_ai.py → test_pydantic_ai_wrap_openai.py} +1 -2
- {braintrust-0.3.14.dist-info → braintrust-0.4.0.dist-info}/METADATA +3 -2
- braintrust-0.4.0.dist-info/RECORD +120 -0
- braintrust-0.3.14.dist-info/RECORD +0 -117
- {braintrust-0.3.14.dist-info → braintrust-0.4.0.dist-info}/WHEEL +0 -0
- {braintrust-0.3.14.dist-info → braintrust-0.4.0.dist-info}/entry_points.txt +0 -0
- {braintrust-0.3.14.dist-info → braintrust-0.4.0.dist-info}/top_level.txt +0 -0
braintrust/_generated_types.py
CHANGED
|
@@ -6,11 +6,12 @@ Auto-generated file -- do not modify.
|
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
-
from
|
|
9
|
+
from collections.abc import Mapping, Sequence
|
|
10
|
+
from typing import Any, Literal, TypeAlias, TypedDict
|
|
10
11
|
|
|
11
12
|
from typing_extensions import NotRequired
|
|
12
13
|
|
|
13
|
-
AclObjectType = Literal[
|
|
14
|
+
AclObjectType: TypeAlias = Literal[
|
|
14
15
|
'organization',
|
|
15
16
|
'project',
|
|
16
17
|
'experiment',
|
|
@@ -23,6 +24,9 @@ AclObjectType = Literal[
|
|
|
23
24
|
'project_log',
|
|
24
25
|
'org_project',
|
|
25
26
|
]
|
|
27
|
+
"""
|
|
28
|
+
The object type that the ACL applies to
|
|
29
|
+
"""
|
|
26
30
|
|
|
27
31
|
|
|
28
32
|
class AISecret(TypedDict):
|
|
@@ -30,11 +34,11 @@ class AISecret(TypedDict):
|
|
|
30
34
|
"""
|
|
31
35
|
Unique identifier for the AI secret
|
|
32
36
|
"""
|
|
33
|
-
created: NotRequired[
|
|
37
|
+
created: NotRequired[str | None]
|
|
34
38
|
"""
|
|
35
39
|
Date of AI secret creation
|
|
36
40
|
"""
|
|
37
|
-
updated_at: NotRequired[
|
|
41
|
+
updated_at: NotRequired[str | None]
|
|
38
42
|
"""
|
|
39
43
|
Date of last AI secret update
|
|
40
44
|
"""
|
|
@@ -46,9 +50,9 @@ class AISecret(TypedDict):
|
|
|
46
50
|
"""
|
|
47
51
|
Name of the AI secret
|
|
48
52
|
"""
|
|
49
|
-
type: NotRequired[
|
|
50
|
-
metadata: NotRequired[
|
|
51
|
-
preview_secret: NotRequired[
|
|
53
|
+
type: NotRequired[str | None]
|
|
54
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
55
|
+
preview_secret: NotRequired[str | None]
|
|
52
56
|
|
|
53
57
|
|
|
54
58
|
class AnyModelParamsToolChoiceFunction(TypedDict):
|
|
@@ -69,7 +73,7 @@ class ApiKey(TypedDict):
|
|
|
69
73
|
"""
|
|
70
74
|
Unique identifier for the api key
|
|
71
75
|
"""
|
|
72
|
-
created: NotRequired[
|
|
76
|
+
created: NotRequired[str | None]
|
|
73
77
|
"""
|
|
74
78
|
Date of api key creation
|
|
75
79
|
"""
|
|
@@ -78,23 +82,23 @@ class ApiKey(TypedDict):
|
|
|
78
82
|
Name of the api key
|
|
79
83
|
"""
|
|
80
84
|
preview_name: str
|
|
81
|
-
user_id: NotRequired[
|
|
85
|
+
user_id: NotRequired[str | None]
|
|
82
86
|
"""
|
|
83
87
|
Unique identifier for the user
|
|
84
88
|
"""
|
|
85
|
-
user_email: NotRequired[
|
|
89
|
+
user_email: NotRequired[str | None]
|
|
86
90
|
"""
|
|
87
91
|
The user's email
|
|
88
92
|
"""
|
|
89
|
-
user_given_name: NotRequired[
|
|
93
|
+
user_given_name: NotRequired[str | None]
|
|
90
94
|
"""
|
|
91
95
|
Given name of the user
|
|
92
96
|
"""
|
|
93
|
-
user_family_name: NotRequired[
|
|
97
|
+
user_family_name: NotRequired[str | None]
|
|
94
98
|
"""
|
|
95
99
|
Family name of the user
|
|
96
100
|
"""
|
|
97
|
-
org_id: NotRequired[
|
|
101
|
+
org_id: NotRequired[str | None]
|
|
98
102
|
"""
|
|
99
103
|
Unique identifier for the organization
|
|
100
104
|
"""
|
|
@@ -116,15 +120,15 @@ class AsyncScoringControlAsyncScoringControl3(TypedDict):
|
|
|
116
120
|
class AsyncScoringStateAsyncScoringState(TypedDict):
|
|
117
121
|
status: Literal['enabled']
|
|
118
122
|
token: str
|
|
119
|
-
function_ids: Sequence
|
|
120
|
-
skip_logging: NotRequired[
|
|
123
|
+
function_ids: Sequence[Any]
|
|
124
|
+
skip_logging: NotRequired[bool | None]
|
|
121
125
|
|
|
122
126
|
|
|
123
127
|
class AsyncScoringStateAsyncScoringState1(TypedDict):
|
|
124
128
|
status: Literal['disabled']
|
|
125
129
|
|
|
126
130
|
|
|
127
|
-
AsyncScoringState =
|
|
131
|
+
AsyncScoringState: TypeAlias = AsyncScoringStateAsyncScoringState | AsyncScoringStateAsyncScoringState1 | None
|
|
128
132
|
|
|
129
133
|
|
|
130
134
|
class BraintrustAttachmentReference(TypedDict):
|
|
@@ -147,75 +151,75 @@ class BraintrustAttachmentReference(TypedDict):
|
|
|
147
151
|
|
|
148
152
|
|
|
149
153
|
class BraintrustModelParams(TypedDict):
|
|
150
|
-
use_cache: NotRequired[
|
|
151
|
-
reasoning_enabled: NotRequired[
|
|
152
|
-
reasoning_budget: NotRequired[
|
|
154
|
+
use_cache: NotRequired[bool | None]
|
|
155
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
156
|
+
reasoning_budget: NotRequired[float | None]
|
|
153
157
|
|
|
154
158
|
|
|
155
159
|
class CallEventCallEvent(TypedDict):
|
|
156
|
-
id: NotRequired[
|
|
160
|
+
id: NotRequired[str | None]
|
|
157
161
|
data: str
|
|
158
162
|
event: Literal['text_delta']
|
|
159
163
|
|
|
160
164
|
|
|
161
165
|
class CallEventCallEvent1(TypedDict):
|
|
162
|
-
id: NotRequired[
|
|
166
|
+
id: NotRequired[str | None]
|
|
163
167
|
data: str
|
|
164
168
|
event: Literal['reasoning_delta']
|
|
165
169
|
|
|
166
170
|
|
|
167
171
|
class CallEventCallEvent2(TypedDict):
|
|
168
|
-
id: NotRequired[
|
|
172
|
+
id: NotRequired[str | None]
|
|
169
173
|
data: str
|
|
170
174
|
event: Literal['json_delta']
|
|
171
175
|
|
|
172
176
|
|
|
173
177
|
class CallEventCallEvent3(TypedDict):
|
|
174
|
-
id: NotRequired[
|
|
178
|
+
id: NotRequired[str | None]
|
|
175
179
|
data: str
|
|
176
180
|
event: Literal['progress']
|
|
177
181
|
|
|
178
182
|
|
|
179
183
|
class CallEventCallEvent4(TypedDict):
|
|
180
|
-
id: NotRequired[
|
|
184
|
+
id: NotRequired[str | None]
|
|
181
185
|
data: str
|
|
182
186
|
event: Literal['error']
|
|
183
187
|
|
|
184
188
|
|
|
185
189
|
class CallEventCallEvent5(TypedDict):
|
|
186
|
-
id: NotRequired[
|
|
190
|
+
id: NotRequired[str | None]
|
|
187
191
|
data: str
|
|
188
192
|
event: Literal['console']
|
|
189
193
|
|
|
190
194
|
|
|
191
195
|
class CallEventCallEvent6(TypedDict):
|
|
192
|
-
id: NotRequired[
|
|
196
|
+
id: NotRequired[str | None]
|
|
193
197
|
event: Literal['start']
|
|
194
198
|
data: Literal['']
|
|
195
199
|
|
|
196
200
|
|
|
197
201
|
class CallEventCallEvent7(TypedDict):
|
|
198
|
-
id: NotRequired[
|
|
202
|
+
id: NotRequired[str | None]
|
|
199
203
|
event: Literal['done']
|
|
200
204
|
data: Literal['']
|
|
201
205
|
|
|
202
206
|
|
|
203
|
-
CallEvent =
|
|
204
|
-
CallEventCallEvent
|
|
205
|
-
CallEventCallEvent1
|
|
206
|
-
CallEventCallEvent2
|
|
207
|
-
CallEventCallEvent3
|
|
208
|
-
CallEventCallEvent4
|
|
209
|
-
CallEventCallEvent5
|
|
210
|
-
CallEventCallEvent6
|
|
211
|
-
CallEventCallEvent7
|
|
212
|
-
|
|
207
|
+
CallEvent: TypeAlias = (
|
|
208
|
+
CallEventCallEvent
|
|
209
|
+
| CallEventCallEvent1
|
|
210
|
+
| CallEventCallEvent2
|
|
211
|
+
| CallEventCallEvent3
|
|
212
|
+
| CallEventCallEvent4
|
|
213
|
+
| CallEventCallEvent5
|
|
214
|
+
| CallEventCallEvent6
|
|
215
|
+
| CallEventCallEvent7
|
|
216
|
+
)
|
|
213
217
|
|
|
214
218
|
|
|
215
219
|
class ChatCompletionContentPartFileFile(TypedDict):
|
|
216
|
-
file_data: NotRequired[
|
|
217
|
-
filename: NotRequired[
|
|
218
|
-
file_id: NotRequired[
|
|
220
|
+
file_data: NotRequired[str | None]
|
|
221
|
+
filename: NotRequired[str | None]
|
|
222
|
+
file_id: NotRequired[str | None]
|
|
219
223
|
|
|
220
224
|
|
|
221
225
|
class ChatCompletionContentPartFileWithTitle(TypedDict):
|
|
@@ -225,7 +229,7 @@ class ChatCompletionContentPartFileWithTitle(TypedDict):
|
|
|
225
229
|
|
|
226
230
|
class ChatCompletionContentPartImageWithTitleImageUrl(TypedDict):
|
|
227
231
|
url: str
|
|
228
|
-
detail: NotRequired[
|
|
232
|
+
detail: NotRequired[Literal['auto'] | Literal['low'] | Literal['high'] | None]
|
|
229
233
|
|
|
230
234
|
|
|
231
235
|
class ChatCompletionContentPartImageWithTitle(TypedDict):
|
|
@@ -240,7 +244,7 @@ class ChatCompletionContentPartTextCacheControl(TypedDict):
|
|
|
240
244
|
class ChatCompletionContentPartText(TypedDict):
|
|
241
245
|
text: str
|
|
242
246
|
type: Literal['text']
|
|
243
|
-
cache_control: NotRequired[
|
|
247
|
+
cache_control: NotRequired[ChatCompletionContentPartTextCacheControl | None]
|
|
244
248
|
|
|
245
249
|
|
|
246
250
|
class ChatCompletionContentPartTextWithTitleCacheControl(TypedDict):
|
|
@@ -250,13 +254,13 @@ class ChatCompletionContentPartTextWithTitleCacheControl(TypedDict):
|
|
|
250
254
|
class ChatCompletionContentPartTextWithTitle(TypedDict):
|
|
251
255
|
text: str
|
|
252
256
|
type: Literal['text']
|
|
253
|
-
cache_control: NotRequired[
|
|
257
|
+
cache_control: NotRequired[ChatCompletionContentPartTextWithTitleCacheControl | None]
|
|
254
258
|
|
|
255
259
|
|
|
256
260
|
class ChatCompletionMessageParamChatCompletionMessageParam(TypedDict):
|
|
257
|
-
content:
|
|
261
|
+
content: str | Sequence[ChatCompletionContentPartText]
|
|
258
262
|
role: Literal['system']
|
|
259
|
-
name: NotRequired[
|
|
263
|
+
name: NotRequired[str | None]
|
|
260
264
|
|
|
261
265
|
|
|
262
266
|
class ChatCompletionMessageParamChatCompletionMessageParam2FunctionCall(TypedDict):
|
|
@@ -265,31 +269,31 @@ class ChatCompletionMessageParamChatCompletionMessageParam2FunctionCall(TypedDic
|
|
|
265
269
|
|
|
266
270
|
|
|
267
271
|
class ChatCompletionMessageParamChatCompletionMessageParam3(TypedDict):
|
|
268
|
-
content:
|
|
272
|
+
content: str | Sequence[ChatCompletionContentPartText]
|
|
269
273
|
role: Literal['tool']
|
|
270
274
|
tool_call_id: str
|
|
271
275
|
|
|
272
276
|
|
|
273
277
|
class ChatCompletionMessageParamChatCompletionMessageParam4(TypedDict):
|
|
274
|
-
content:
|
|
278
|
+
content: str | None
|
|
275
279
|
name: str
|
|
276
280
|
role: Literal['function']
|
|
277
281
|
|
|
278
282
|
|
|
279
283
|
class ChatCompletionMessageParamChatCompletionMessageParam5(TypedDict):
|
|
280
|
-
content:
|
|
284
|
+
content: str | Sequence[ChatCompletionContentPartText]
|
|
281
285
|
role: Literal['developer']
|
|
282
|
-
name: NotRequired[
|
|
286
|
+
name: NotRequired[str | None]
|
|
283
287
|
|
|
284
288
|
|
|
285
289
|
class ChatCompletionMessageParamChatCompletionMessageParam6(TypedDict):
|
|
286
290
|
role: Literal['model']
|
|
287
|
-
content: NotRequired[
|
|
291
|
+
content: NotRequired[str | None]
|
|
288
292
|
|
|
289
293
|
|
|
290
294
|
class ChatCompletionMessageReasoning(TypedDict):
|
|
291
|
-
id: NotRequired[
|
|
292
|
-
content: NotRequired[
|
|
295
|
+
id: NotRequired[str | None]
|
|
296
|
+
content: NotRequired[str | None]
|
|
293
297
|
|
|
294
298
|
|
|
295
299
|
class ChatCompletionMessageToolCallFunction(TypedDict):
|
|
@@ -304,9 +308,9 @@ class ChatCompletionMessageToolCall(TypedDict):
|
|
|
304
308
|
|
|
305
309
|
|
|
306
310
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam(TypedDict):
|
|
307
|
-
content:
|
|
311
|
+
content: str | Sequence[ChatCompletionContentPartText]
|
|
308
312
|
role: Literal['system']
|
|
309
|
-
name: NotRequired[
|
|
313
|
+
name: NotRequired[str | None]
|
|
310
314
|
|
|
311
315
|
|
|
312
316
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2FunctionCall(TypedDict):
|
|
@@ -316,35 +320,35 @@ class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2FunctionC
|
|
|
316
320
|
|
|
317
321
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2(TypedDict):
|
|
318
322
|
role: Literal['assistant']
|
|
319
|
-
content: NotRequired[
|
|
320
|
-
function_call: NotRequired[
|
|
321
|
-
name: NotRequired[
|
|
322
|
-
tool_calls: NotRequired[
|
|
323
|
-
reasoning: NotRequired[
|
|
323
|
+
content: NotRequired[str | Sequence[ChatCompletionContentPartText] | None]
|
|
324
|
+
function_call: NotRequired[ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2FunctionCall | None]
|
|
325
|
+
name: NotRequired[str | None]
|
|
326
|
+
tool_calls: NotRequired[Sequence[ChatCompletionMessageToolCall] | None]
|
|
327
|
+
reasoning: NotRequired[Sequence[ChatCompletionMessageReasoning] | None]
|
|
324
328
|
|
|
325
329
|
|
|
326
330
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam3(TypedDict):
|
|
327
|
-
content:
|
|
331
|
+
content: str | Sequence[ChatCompletionContentPartText]
|
|
328
332
|
role: Literal['tool']
|
|
329
333
|
tool_call_id: str
|
|
330
334
|
|
|
331
335
|
|
|
332
336
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam4(TypedDict):
|
|
333
|
-
content:
|
|
337
|
+
content: str | None
|
|
334
338
|
name: str
|
|
335
339
|
role: Literal['function']
|
|
336
340
|
|
|
337
341
|
|
|
338
342
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam5(TypedDict):
|
|
339
|
-
content:
|
|
343
|
+
content: str | Sequence[ChatCompletionContentPartText]
|
|
340
344
|
role: Literal['developer']
|
|
341
|
-
name: NotRequired[
|
|
345
|
+
name: NotRequired[str | None]
|
|
342
346
|
|
|
343
347
|
|
|
344
348
|
class ChatCompletionToolFunction(TypedDict):
|
|
345
349
|
name: str
|
|
346
|
-
description: NotRequired[
|
|
347
|
-
parameters: NotRequired[
|
|
350
|
+
description: NotRequired[str | None]
|
|
351
|
+
parameters: NotRequired[Mapping[str, Any] | None]
|
|
348
352
|
|
|
349
353
|
|
|
350
354
|
class ChatCompletionTool(TypedDict):
|
|
@@ -353,7 +357,7 @@ class ChatCompletionTool(TypedDict):
|
|
|
353
357
|
|
|
354
358
|
|
|
355
359
|
class CodeBundleRuntimeContext(TypedDict):
|
|
356
|
-
runtime: Literal['node', 'python', 'browser']
|
|
360
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
357
361
|
version: str
|
|
358
362
|
|
|
359
363
|
|
|
@@ -369,7 +373,7 @@ class CodeBundleLocationPosition1(TypedDict):
|
|
|
369
373
|
class CodeBundleLocation(TypedDict):
|
|
370
374
|
type: Literal['experiment']
|
|
371
375
|
eval_name: str
|
|
372
|
-
position:
|
|
376
|
+
position: CodeBundleLocationPosition | CodeBundleLocationPosition1
|
|
373
377
|
|
|
374
378
|
|
|
375
379
|
class CodeBundleLocation1(TypedDict):
|
|
@@ -379,9 +383,9 @@ class CodeBundleLocation1(TypedDict):
|
|
|
379
383
|
|
|
380
384
|
class CodeBundle(TypedDict):
|
|
381
385
|
runtime_context: CodeBundleRuntimeContext
|
|
382
|
-
location:
|
|
386
|
+
location: CodeBundleLocation | CodeBundleLocation1
|
|
383
387
|
bundle_id: str
|
|
384
|
-
preview: NotRequired[
|
|
388
|
+
preview: NotRequired[str | None]
|
|
385
389
|
"""
|
|
386
390
|
A preview of the code
|
|
387
391
|
"""
|
|
@@ -400,30 +404,30 @@ class Dataset(TypedDict):
|
|
|
400
404
|
"""
|
|
401
405
|
Name of the dataset. Within a project, dataset names are unique
|
|
402
406
|
"""
|
|
403
|
-
description: NotRequired[
|
|
407
|
+
description: NotRequired[str | None]
|
|
404
408
|
"""
|
|
405
409
|
Textual description of the dataset
|
|
406
410
|
"""
|
|
407
|
-
created: NotRequired[
|
|
411
|
+
created: NotRequired[str | None]
|
|
408
412
|
"""
|
|
409
413
|
Date of dataset creation
|
|
410
414
|
"""
|
|
411
|
-
deleted_at: NotRequired[
|
|
415
|
+
deleted_at: NotRequired[str | None]
|
|
412
416
|
"""
|
|
413
417
|
Date of dataset deletion, or null if the dataset is still active
|
|
414
418
|
"""
|
|
415
|
-
user_id: NotRequired[
|
|
419
|
+
user_id: NotRequired[str | None]
|
|
416
420
|
"""
|
|
417
421
|
Identifies the user who created the dataset
|
|
418
422
|
"""
|
|
419
|
-
metadata: NotRequired[
|
|
423
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
420
424
|
"""
|
|
421
425
|
User-controlled metadata about the dataset
|
|
422
426
|
"""
|
|
423
427
|
|
|
424
428
|
|
|
425
429
|
class DatasetEventMetadata(TypedDict):
|
|
426
|
-
model: NotRequired[
|
|
430
|
+
model: NotRequired[str | None]
|
|
427
431
|
"""
|
|
428
432
|
The model used for this example
|
|
429
433
|
"""
|
|
@@ -446,68 +450,80 @@ class EnvVar(TypedDict):
|
|
|
446
450
|
"""
|
|
447
451
|
The name of the environment variable
|
|
448
452
|
"""
|
|
449
|
-
created: NotRequired[
|
|
453
|
+
created: NotRequired[str | None]
|
|
450
454
|
"""
|
|
451
455
|
Date of environment variable creation
|
|
452
456
|
"""
|
|
453
|
-
used: NotRequired[
|
|
457
|
+
used: NotRequired[str | None]
|
|
454
458
|
"""
|
|
455
459
|
Date the environment variable was last used
|
|
456
460
|
"""
|
|
461
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
462
|
+
"""
|
|
463
|
+
Optional metadata associated with the environment variable when managed via the function secrets API
|
|
464
|
+
"""
|
|
465
|
+
secret_type: NotRequired[str | None]
|
|
466
|
+
"""
|
|
467
|
+
Optional classification for the secret (for example, the AI provider name)
|
|
468
|
+
"""
|
|
469
|
+
secret_category: NotRequired[Literal['env_var', 'ai_provider'] | None]
|
|
470
|
+
"""
|
|
471
|
+
The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys
|
|
472
|
+
"""
|
|
457
473
|
|
|
458
474
|
|
|
459
475
|
class ExperimentEventMetadata(TypedDict):
|
|
460
|
-
model: NotRequired[
|
|
476
|
+
model: NotRequired[str | None]
|
|
461
477
|
"""
|
|
462
478
|
The model used for this example
|
|
463
479
|
"""
|
|
464
480
|
|
|
465
481
|
|
|
466
482
|
class ExperimentEventMetrics(TypedDict):
|
|
467
|
-
start: NotRequired[
|
|
483
|
+
start: NotRequired[float | None]
|
|
468
484
|
"""
|
|
469
485
|
A unix timestamp recording when the section of code which produced the experiment event started
|
|
470
486
|
"""
|
|
471
|
-
end: NotRequired[
|
|
487
|
+
end: NotRequired[float | None]
|
|
472
488
|
"""
|
|
473
489
|
A unix timestamp recording when the section of code which produced the experiment event finished
|
|
474
490
|
"""
|
|
475
|
-
prompt_tokens: NotRequired[
|
|
491
|
+
prompt_tokens: NotRequired[int | None]
|
|
476
492
|
"""
|
|
477
493
|
The number of tokens in the prompt used to generate the experiment event (only set if this is an LLM span)
|
|
478
494
|
"""
|
|
479
|
-
completion_tokens: NotRequired[
|
|
495
|
+
completion_tokens: NotRequired[int | None]
|
|
480
496
|
"""
|
|
481
497
|
The number of tokens in the completion generated by the model (only set if this is an LLM span)
|
|
482
498
|
"""
|
|
483
|
-
tokens: NotRequired[
|
|
499
|
+
tokens: NotRequired[int | None]
|
|
484
500
|
"""
|
|
485
501
|
The total number of tokens in the input and output of the experiment event.
|
|
486
502
|
"""
|
|
487
|
-
caller_functionname: NotRequired[
|
|
503
|
+
caller_functionname: NotRequired[Any | None]
|
|
488
504
|
"""
|
|
489
505
|
This metric is deprecated
|
|
490
506
|
"""
|
|
491
|
-
caller_filename: NotRequired[
|
|
507
|
+
caller_filename: NotRequired[Any | None]
|
|
492
508
|
"""
|
|
493
509
|
This metric is deprecated
|
|
494
510
|
"""
|
|
495
|
-
caller_lineno: NotRequired[
|
|
511
|
+
caller_lineno: NotRequired[Any | None]
|
|
496
512
|
"""
|
|
497
513
|
This metric is deprecated
|
|
498
514
|
"""
|
|
499
515
|
|
|
500
516
|
|
|
501
517
|
class ExperimentEventContext(TypedDict):
|
|
502
|
-
caller_functionname: NotRequired[
|
|
518
|
+
caller_functionname: NotRequired[str | None]
|
|
503
519
|
"""
|
|
504
520
|
The function in code which created the experiment event
|
|
505
521
|
"""
|
|
506
|
-
caller_filename: NotRequired[
|
|
522
|
+
caller_filename: NotRequired[str | None]
|
|
507
523
|
"""
|
|
508
524
|
Name of the file in code where the experiment event was created
|
|
509
525
|
"""
|
|
510
|
-
caller_lineno: NotRequired[
|
|
526
|
+
caller_lineno: NotRequired[int | None]
|
|
511
527
|
"""
|
|
512
528
|
Line of code where the experiment event was created
|
|
513
529
|
"""
|
|
@@ -518,24 +534,12 @@ class ExtendedSavedFunctionIdExtendedSavedFunctionId(TypedDict):
|
|
|
518
534
|
id: str
|
|
519
535
|
|
|
520
536
|
|
|
521
|
-
class ExtendedSavedFunctionIdExtendedSavedFunctionId1(TypedDict):
|
|
522
|
-
type: Literal['global']
|
|
523
|
-
name: str
|
|
524
|
-
|
|
525
|
-
|
|
526
537
|
class ExtendedSavedFunctionIdExtendedSavedFunctionId2(TypedDict):
|
|
527
538
|
type: Literal['slug']
|
|
528
539
|
project_id: str
|
|
529
540
|
slug: str
|
|
530
541
|
|
|
531
542
|
|
|
532
|
-
ExtendedSavedFunctionId = Union[
|
|
533
|
-
ExtendedSavedFunctionIdExtendedSavedFunctionId,
|
|
534
|
-
ExtendedSavedFunctionIdExtendedSavedFunctionId1,
|
|
535
|
-
ExtendedSavedFunctionIdExtendedSavedFunctionId2,
|
|
536
|
-
]
|
|
537
|
-
|
|
538
|
-
|
|
539
543
|
class ExternalAttachmentReference(TypedDict):
|
|
540
544
|
type: Literal['external_attachment']
|
|
541
545
|
"""
|
|
@@ -555,21 +559,34 @@ class ExternalAttachmentReference(TypedDict):
|
|
|
555
559
|
"""
|
|
556
560
|
|
|
557
561
|
|
|
562
|
+
class PreprocessorPreprocessor(TypedDict):
|
|
563
|
+
type: Literal['function']
|
|
564
|
+
id: str
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
class PreprocessorPreprocessor2(TypedDict):
|
|
568
|
+
pass
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
class PreprocessorPreprocessor3(PreprocessorPreprocessor, PreprocessorPreprocessor2):
|
|
572
|
+
pass
|
|
573
|
+
|
|
574
|
+
|
|
558
575
|
class FunctionOrigin(TypedDict):
|
|
559
|
-
object_type:
|
|
576
|
+
object_type: AclObjectType
|
|
560
577
|
object_id: str
|
|
561
578
|
"""
|
|
562
579
|
Id of the object the function is originating from
|
|
563
580
|
"""
|
|
564
|
-
internal: NotRequired[
|
|
581
|
+
internal: NotRequired[bool | None]
|
|
565
582
|
"""
|
|
566
583
|
The function exists for internal purposes and should not be displayed in the list of functions.
|
|
567
584
|
"""
|
|
568
585
|
|
|
569
586
|
|
|
570
587
|
class FunctionFunctionSchema(TypedDict):
|
|
571
|
-
parameters: NotRequired[
|
|
572
|
-
returns: NotRequired[
|
|
588
|
+
parameters: NotRequired[Any | None]
|
|
589
|
+
returns: NotRequired[Any | None]
|
|
573
590
|
|
|
574
591
|
|
|
575
592
|
class FunctionDataFunctionData(TypedDict):
|
|
@@ -581,7 +598,7 @@ class Data(CodeBundle):
|
|
|
581
598
|
|
|
582
599
|
|
|
583
600
|
class FunctionDataFunctionData1DataRuntimeContext(TypedDict):
|
|
584
|
-
runtime: Literal['node', 'python', 'browser']
|
|
601
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
585
602
|
version: str
|
|
586
603
|
|
|
587
604
|
|
|
@@ -589,11 +606,15 @@ class FunctionDataFunctionData1Data(TypedDict):
|
|
|
589
606
|
type: Literal['inline']
|
|
590
607
|
runtime_context: FunctionDataFunctionData1DataRuntimeContext
|
|
591
608
|
code: str
|
|
609
|
+
code_hash: NotRequired[str | None]
|
|
610
|
+
"""
|
|
611
|
+
SHA256 hash of the code, computed at save time
|
|
612
|
+
"""
|
|
592
613
|
|
|
593
614
|
|
|
594
615
|
class FunctionDataFunctionData1(TypedDict):
|
|
595
616
|
type: Literal['code']
|
|
596
|
-
data:
|
|
617
|
+
data: Data | FunctionDataFunctionData1Data
|
|
597
618
|
|
|
598
619
|
|
|
599
620
|
class FunctionDataFunctionData2(TypedDict):
|
|
@@ -603,12 +624,7 @@ class FunctionDataFunctionData2(TypedDict):
|
|
|
603
624
|
parameters: Mapping[str, Any]
|
|
604
625
|
|
|
605
626
|
|
|
606
|
-
|
|
607
|
-
type: Literal['global']
|
|
608
|
-
name: str
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
FunctionFormat = Literal['llm', 'code', 'global', 'graph']
|
|
627
|
+
FunctionFormat: TypeAlias = Literal['llm', 'code', 'global', 'graph']
|
|
612
628
|
|
|
613
629
|
|
|
614
630
|
class FunctionIdFunctionId(TypedDict):
|
|
@@ -616,7 +632,7 @@ class FunctionIdFunctionId(TypedDict):
|
|
|
616
632
|
"""
|
|
617
633
|
The ID of the function
|
|
618
634
|
"""
|
|
619
|
-
version: NotRequired[
|
|
635
|
+
version: NotRequired[str | None]
|
|
620
636
|
"""
|
|
621
637
|
The version of the function
|
|
622
638
|
"""
|
|
@@ -631,19 +647,12 @@ class FunctionIdFunctionId1(TypedDict):
|
|
|
631
647
|
"""
|
|
632
648
|
The slug of the function
|
|
633
649
|
"""
|
|
634
|
-
version: NotRequired[
|
|
650
|
+
version: NotRequired[str | None]
|
|
635
651
|
"""
|
|
636
652
|
The version of the function
|
|
637
653
|
"""
|
|
638
654
|
|
|
639
655
|
|
|
640
|
-
class FunctionIdFunctionId2(TypedDict):
|
|
641
|
-
global_function: str
|
|
642
|
-
"""
|
|
643
|
-
The name of the global function. Currently, the global namespace includes the functions in autoevals
|
|
644
|
-
"""
|
|
645
|
-
|
|
646
|
-
|
|
647
656
|
class FunctionIdFunctionId3(TypedDict):
|
|
648
657
|
prompt_session_id: str
|
|
649
658
|
"""
|
|
@@ -653,14 +662,14 @@ class FunctionIdFunctionId3(TypedDict):
|
|
|
653
662
|
"""
|
|
654
663
|
The ID of the function in the prompt session
|
|
655
664
|
"""
|
|
656
|
-
version: NotRequired[
|
|
665
|
+
version: NotRequired[str | None]
|
|
657
666
|
"""
|
|
658
667
|
The version of the function
|
|
659
668
|
"""
|
|
660
669
|
|
|
661
670
|
|
|
662
671
|
class FunctionIdFunctionId4InlineContext(TypedDict):
|
|
663
|
-
runtime: Literal['node', 'python', 'browser']
|
|
672
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
664
673
|
version: str
|
|
665
674
|
|
|
666
675
|
|
|
@@ -670,25 +679,27 @@ class FunctionIdFunctionId4(TypedDict):
|
|
|
670
679
|
"""
|
|
671
680
|
The inline code to execute
|
|
672
681
|
"""
|
|
673
|
-
name: NotRequired[
|
|
682
|
+
name: NotRequired[str | None]
|
|
674
683
|
"""
|
|
675
684
|
The name of the inline code function
|
|
676
685
|
"""
|
|
677
686
|
|
|
678
687
|
|
|
679
|
-
FunctionIdRef = Mapping[str, Any]
|
|
688
|
+
FunctionIdRef: TypeAlias = Mapping[str, Any]
|
|
680
689
|
|
|
681
690
|
|
|
682
|
-
FunctionObjectType = Literal[
|
|
691
|
+
FunctionObjectType: TypeAlias = Literal[
|
|
692
|
+
'prompt', 'tool', 'scorer', 'task', 'agent', 'custom_view', 'preprocessor', 'facet'
|
|
693
|
+
]
|
|
683
694
|
|
|
684
695
|
|
|
685
|
-
FunctionOutputType = Literal['completion', 'score', 'any']
|
|
696
|
+
FunctionOutputType: TypeAlias = Literal['completion', 'score', 'any']
|
|
686
697
|
|
|
687
698
|
|
|
688
|
-
FunctionTypeEnum = Literal['llm', 'scorer', 'task', 'tool', 'custom_view']
|
|
699
|
+
FunctionTypeEnum: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet']
|
|
689
700
|
|
|
690
701
|
|
|
691
|
-
FunctionTypeEnumNullish = Literal['llm', 'scorer', 'task', 'tool', 'custom_view']
|
|
702
|
+
FunctionTypeEnumNullish: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet']
|
|
692
703
|
|
|
693
704
|
|
|
694
705
|
class GitMetadataSettings(TypedDict):
|
|
@@ -747,11 +758,11 @@ class GraphNodeGraphNodePosition(TypedDict):
|
|
|
747
758
|
|
|
748
759
|
|
|
749
760
|
class GraphNodeGraphNode(TypedDict):
|
|
750
|
-
description: NotRequired[
|
|
761
|
+
description: NotRequired[str | None]
|
|
751
762
|
"""
|
|
752
763
|
The description of the node
|
|
753
764
|
"""
|
|
754
|
-
position: NotRequired[
|
|
765
|
+
position: NotRequired[GraphNodeGraphNodePosition | None]
|
|
755
766
|
"""
|
|
756
767
|
The position of the node
|
|
757
768
|
"""
|
|
@@ -771,11 +782,11 @@ class GraphNodeGraphNode1Position(TypedDict):
|
|
|
771
782
|
|
|
772
783
|
|
|
773
784
|
class GraphNodeGraphNode1(TypedDict):
|
|
774
|
-
description: NotRequired[
|
|
785
|
+
description: NotRequired[str | None]
|
|
775
786
|
"""
|
|
776
787
|
The description of the node
|
|
777
788
|
"""
|
|
778
|
-
position: NotRequired[
|
|
789
|
+
position: NotRequired[GraphNodeGraphNode1Position | None]
|
|
779
790
|
"""
|
|
780
791
|
The position of the node
|
|
781
792
|
"""
|
|
@@ -797,11 +808,11 @@ class GraphNodeGraphNode2Position(TypedDict):
|
|
|
797
808
|
|
|
798
809
|
|
|
799
810
|
class GraphNodeGraphNode2(TypedDict):
|
|
800
|
-
description: NotRequired[
|
|
811
|
+
description: NotRequired[str | None]
|
|
801
812
|
"""
|
|
802
813
|
The description of the node
|
|
803
814
|
"""
|
|
804
|
-
position: NotRequired[
|
|
815
|
+
position: NotRequired[GraphNodeGraphNode2Position | None]
|
|
805
816
|
"""
|
|
806
817
|
The position of the node
|
|
807
818
|
"""
|
|
@@ -823,16 +834,16 @@ class GraphNodeGraphNode3Position(TypedDict):
|
|
|
823
834
|
|
|
824
835
|
|
|
825
836
|
class GraphNodeGraphNode3(TypedDict):
|
|
826
|
-
description: NotRequired[
|
|
837
|
+
description: NotRequired[str | None]
|
|
827
838
|
"""
|
|
828
839
|
The description of the node
|
|
829
840
|
"""
|
|
830
|
-
position: NotRequired[
|
|
841
|
+
position: NotRequired[GraphNodeGraphNode3Position | None]
|
|
831
842
|
"""
|
|
832
843
|
The position of the node
|
|
833
844
|
"""
|
|
834
845
|
type: Literal['literal']
|
|
835
|
-
value: NotRequired[
|
|
846
|
+
value: NotRequired[Any | None]
|
|
836
847
|
"""
|
|
837
848
|
A literal value to be returned
|
|
838
849
|
"""
|
|
@@ -850,11 +861,11 @@ class GraphNodeGraphNode4Position(TypedDict):
|
|
|
850
861
|
|
|
851
862
|
|
|
852
863
|
class GraphNodeGraphNode4(TypedDict):
|
|
853
|
-
description: NotRequired[
|
|
864
|
+
description: NotRequired[str | None]
|
|
854
865
|
"""
|
|
855
866
|
The description of the node
|
|
856
867
|
"""
|
|
857
|
-
position: NotRequired[
|
|
868
|
+
position: NotRequired[GraphNodeGraphNode4Position | None]
|
|
858
869
|
"""
|
|
859
870
|
The position of the node
|
|
860
871
|
"""
|
|
@@ -877,16 +888,16 @@ class GraphNodeGraphNode5Position(TypedDict):
|
|
|
877
888
|
|
|
878
889
|
|
|
879
890
|
class GraphNodeGraphNode5(TypedDict):
|
|
880
|
-
description: NotRequired[
|
|
891
|
+
description: NotRequired[str | None]
|
|
881
892
|
"""
|
|
882
893
|
The description of the node
|
|
883
894
|
"""
|
|
884
|
-
position: NotRequired[
|
|
895
|
+
position: NotRequired[GraphNodeGraphNode5Position | None]
|
|
885
896
|
"""
|
|
886
897
|
The position of the node
|
|
887
898
|
"""
|
|
888
899
|
type: Literal['gate']
|
|
889
|
-
condition: NotRequired[
|
|
900
|
+
condition: NotRequired[str | None]
|
|
890
901
|
"""
|
|
891
902
|
A BTQL expression to be evaluated
|
|
892
903
|
"""
|
|
@@ -904,11 +915,11 @@ class GraphNodeGraphNode6Position(TypedDict):
|
|
|
904
915
|
|
|
905
916
|
|
|
906
917
|
class GraphNodeGraphNode6(TypedDict):
|
|
907
|
-
description: NotRequired[
|
|
918
|
+
description: NotRequired[str | None]
|
|
908
919
|
"""
|
|
909
920
|
The description of the node
|
|
910
921
|
"""
|
|
911
|
-
position: NotRequired[
|
|
922
|
+
position: NotRequired[GraphNodeGraphNode6Position | None]
|
|
912
923
|
"""
|
|
913
924
|
The position of the node
|
|
914
925
|
"""
|
|
@@ -937,11 +948,11 @@ class Group(TypedDict):
|
|
|
937
948
|
|
|
938
949
|
It is forbidden to change the org after creating a group
|
|
939
950
|
"""
|
|
940
|
-
user_id: NotRequired[
|
|
951
|
+
user_id: NotRequired[str | None]
|
|
941
952
|
"""
|
|
942
953
|
Identifies the user who created the group
|
|
943
954
|
"""
|
|
944
|
-
created: NotRequired[
|
|
955
|
+
created: NotRequired[str | None]
|
|
945
956
|
"""
|
|
946
957
|
Date of group creation
|
|
947
958
|
"""
|
|
@@ -949,19 +960,19 @@ class Group(TypedDict):
|
|
|
949
960
|
"""
|
|
950
961
|
Name of the group
|
|
951
962
|
"""
|
|
952
|
-
description: NotRequired[
|
|
963
|
+
description: NotRequired[str | None]
|
|
953
964
|
"""
|
|
954
965
|
Textual description of the group
|
|
955
966
|
"""
|
|
956
|
-
deleted_at: NotRequired[
|
|
967
|
+
deleted_at: NotRequired[str | None]
|
|
957
968
|
"""
|
|
958
969
|
Date of group deletion, or null if the group is still active
|
|
959
970
|
"""
|
|
960
|
-
member_users: NotRequired[
|
|
971
|
+
member_users: NotRequired[Sequence[str] | None]
|
|
961
972
|
"""
|
|
962
973
|
Ids of users which belong to this group
|
|
963
974
|
"""
|
|
964
|
-
member_groups: NotRequired[
|
|
975
|
+
member_groups: NotRequired[Sequence[str] | None]
|
|
965
976
|
"""
|
|
966
977
|
Ids of the groups this group inherits from
|
|
967
978
|
|
|
@@ -969,7 +980,72 @@ class Group(TypedDict):
|
|
|
969
980
|
"""
|
|
970
981
|
|
|
971
982
|
|
|
972
|
-
IfExists = Literal['error', 'ignore', 'replace']
|
|
983
|
+
IfExists: TypeAlias = Literal['error', 'ignore', 'replace']
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
class InvokeFunctionInvokeFunction(TypedDict):
|
|
987
|
+
function_id: str
|
|
988
|
+
"""
|
|
989
|
+
The ID of the function
|
|
990
|
+
"""
|
|
991
|
+
version: NotRequired[str | None]
|
|
992
|
+
"""
|
|
993
|
+
The version of the function
|
|
994
|
+
"""
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
class InvokeFunctionInvokeFunction1(TypedDict):
|
|
998
|
+
project_name: str
|
|
999
|
+
"""
|
|
1000
|
+
The name of the project containing the function
|
|
1001
|
+
"""
|
|
1002
|
+
slug: str
|
|
1003
|
+
"""
|
|
1004
|
+
The slug of the function
|
|
1005
|
+
"""
|
|
1006
|
+
version: NotRequired[str | None]
|
|
1007
|
+
"""
|
|
1008
|
+
The version of the function
|
|
1009
|
+
"""
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
class InvokeFunctionInvokeFunction3(TypedDict):
|
|
1013
|
+
prompt_session_id: str
|
|
1014
|
+
"""
|
|
1015
|
+
The ID of the prompt session
|
|
1016
|
+
"""
|
|
1017
|
+
prompt_session_function_id: str
|
|
1018
|
+
"""
|
|
1019
|
+
The ID of the function in the prompt session
|
|
1020
|
+
"""
|
|
1021
|
+
version: NotRequired[str | None]
|
|
1022
|
+
"""
|
|
1023
|
+
The version of the function
|
|
1024
|
+
"""
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
class InvokeFunctionInvokeFunction4InlineContext(TypedDict):
|
|
1028
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
1029
|
+
version: str
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
class InvokeFunctionInvokeFunction4(TypedDict):
|
|
1033
|
+
inline_context: InvokeFunctionInvokeFunction4InlineContext
|
|
1034
|
+
code: str
|
|
1035
|
+
"""
|
|
1036
|
+
The inline code to execute
|
|
1037
|
+
"""
|
|
1038
|
+
name: NotRequired[str | None]
|
|
1039
|
+
"""
|
|
1040
|
+
The name of the inline code function
|
|
1041
|
+
"""
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
class InvokeFunctionMcpAuth(TypedDict):
|
|
1045
|
+
oauth_token: NotRequired[str | None]
|
|
1046
|
+
"""
|
|
1047
|
+
The OAuth token to use
|
|
1048
|
+
"""
|
|
973
1049
|
|
|
974
1050
|
|
|
975
1051
|
class InvokeParentInvokeParentRowIds(TypedDict):
|
|
@@ -993,20 +1069,58 @@ class InvokeParentInvokeParent(TypedDict):
|
|
|
993
1069
|
"""
|
|
994
1070
|
The id of the container object you are logging to
|
|
995
1071
|
"""
|
|
996
|
-
row_ids: NotRequired[
|
|
1072
|
+
row_ids: NotRequired[InvokeParentInvokeParentRowIds | None]
|
|
997
1073
|
"""
|
|
998
1074
|
Identifiers for the row to to log a subspan under
|
|
999
1075
|
"""
|
|
1000
|
-
propagated_event: NotRequired[
|
|
1076
|
+
propagated_event: NotRequired[Mapping[str, Any] | None]
|
|
1001
1077
|
"""
|
|
1002
1078
|
Include these properties in every span created under this parent
|
|
1003
1079
|
"""
|
|
1004
1080
|
|
|
1005
1081
|
|
|
1006
|
-
InvokeParent =
|
|
1082
|
+
InvokeParent: TypeAlias = InvokeParentInvokeParent | str
|
|
1083
|
+
"""
|
|
1084
|
+
Options for tracing the function call
|
|
1085
|
+
"""
|
|
1007
1086
|
|
|
1008
1087
|
|
|
1009
|
-
|
|
1088
|
+
class MCPServer(TypedDict):
|
|
1089
|
+
id: str
|
|
1090
|
+
"""
|
|
1091
|
+
Unique identifier for the MCP server
|
|
1092
|
+
"""
|
|
1093
|
+
project_id: str
|
|
1094
|
+
"""
|
|
1095
|
+
Unique identifier for the project that the MCP server belongs under
|
|
1096
|
+
"""
|
|
1097
|
+
user_id: NotRequired[str | None]
|
|
1098
|
+
"""
|
|
1099
|
+
Identifies the user who created the MCP server
|
|
1100
|
+
"""
|
|
1101
|
+
created: NotRequired[str | None]
|
|
1102
|
+
"""
|
|
1103
|
+
Date of MCP server creation
|
|
1104
|
+
"""
|
|
1105
|
+
deleted_at: NotRequired[str | None]
|
|
1106
|
+
"""
|
|
1107
|
+
Date of MCP server deletion, or null if the MCP server is still active
|
|
1108
|
+
"""
|
|
1109
|
+
name: str
|
|
1110
|
+
"""
|
|
1111
|
+
Name of the MCP server. Within a project, MCP server names are unique
|
|
1112
|
+
"""
|
|
1113
|
+
description: NotRequired[str | None]
|
|
1114
|
+
"""
|
|
1115
|
+
Textual description of the MCP server
|
|
1116
|
+
"""
|
|
1117
|
+
url: str
|
|
1118
|
+
"""
|
|
1119
|
+
URL of the MCP server endpoint
|
|
1120
|
+
"""
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
MessageRole: TypeAlias = Literal['system', 'user', 'assistant', 'function', 'tool', 'model', 'developer']
|
|
1010
1124
|
|
|
1011
1125
|
|
|
1012
1126
|
class ModelParamsModelParamsToolChoiceFunction(TypedDict):
|
|
@@ -1023,42 +1137,67 @@ class ModelParamsModelParamsFunctionCall(TypedDict):
|
|
|
1023
1137
|
|
|
1024
1138
|
|
|
1025
1139
|
class ModelParamsModelParams1(TypedDict):
|
|
1026
|
-
use_cache: NotRequired[
|
|
1027
|
-
reasoning_enabled: NotRequired[
|
|
1028
|
-
reasoning_budget: NotRequired[
|
|
1140
|
+
use_cache: NotRequired[bool | None]
|
|
1141
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
1142
|
+
reasoning_budget: NotRequired[float | None]
|
|
1029
1143
|
max_tokens: float
|
|
1030
1144
|
temperature: float
|
|
1031
|
-
top_p: NotRequired[
|
|
1032
|
-
top_k: NotRequired[
|
|
1033
|
-
stop_sequences: NotRequired[
|
|
1034
|
-
max_tokens_to_sample: NotRequired[
|
|
1145
|
+
top_p: NotRequired[float | None]
|
|
1146
|
+
top_k: NotRequired[float | None]
|
|
1147
|
+
stop_sequences: NotRequired[Sequence[str] | None]
|
|
1148
|
+
max_tokens_to_sample: NotRequired[float | None]
|
|
1035
1149
|
"""
|
|
1036
1150
|
This is a legacy parameter that should not be used.
|
|
1037
1151
|
"""
|
|
1038
1152
|
|
|
1039
1153
|
|
|
1040
1154
|
class ModelParamsModelParams2(TypedDict):
|
|
1041
|
-
use_cache: NotRequired[
|
|
1042
|
-
reasoning_enabled: NotRequired[
|
|
1043
|
-
reasoning_budget: NotRequired[
|
|
1044
|
-
temperature: NotRequired[
|
|
1045
|
-
maxOutputTokens: NotRequired[
|
|
1046
|
-
topP: NotRequired[
|
|
1047
|
-
topK: NotRequired[
|
|
1155
|
+
use_cache: NotRequired[bool | None]
|
|
1156
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
1157
|
+
reasoning_budget: NotRequired[float | None]
|
|
1158
|
+
temperature: NotRequired[float | None]
|
|
1159
|
+
maxOutputTokens: NotRequired[float | None]
|
|
1160
|
+
topP: NotRequired[float | None]
|
|
1161
|
+
topK: NotRequired[float | None]
|
|
1048
1162
|
|
|
1049
1163
|
|
|
1050
1164
|
class ModelParamsModelParams3(TypedDict):
|
|
1051
|
-
use_cache: NotRequired[
|
|
1052
|
-
reasoning_enabled: NotRequired[
|
|
1053
|
-
reasoning_budget: NotRequired[
|
|
1054
|
-
temperature: NotRequired[
|
|
1055
|
-
topK: NotRequired[
|
|
1165
|
+
use_cache: NotRequired[bool | None]
|
|
1166
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
1167
|
+
reasoning_budget: NotRequired[float | None]
|
|
1168
|
+
temperature: NotRequired[float | None]
|
|
1169
|
+
topK: NotRequired[float | None]
|
|
1056
1170
|
|
|
1057
1171
|
|
|
1058
1172
|
class ModelParamsModelParams4(TypedDict):
|
|
1059
|
-
use_cache: NotRequired[
|
|
1060
|
-
reasoning_enabled: NotRequired[
|
|
1061
|
-
reasoning_budget: NotRequired[
|
|
1173
|
+
use_cache: NotRequired[bool | None]
|
|
1174
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
1175
|
+
reasoning_budget: NotRequired[float | None]
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
NullableFunctionTypeEnum: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet']
|
|
1179
|
+
"""
|
|
1180
|
+
The type of global function. If unspecified, defaults to 'scorer' for backward compatibility.
|
|
1181
|
+
"""
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
class NullableSavedFunctionIdNullableSavedFunctionId(TypedDict):
|
|
1185
|
+
type: Literal['function']
|
|
1186
|
+
id: str
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
class NullableSavedFunctionIdNullableSavedFunctionId1(TypedDict):
|
|
1190
|
+
type: Literal['global']
|
|
1191
|
+
name: str
|
|
1192
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
NullableSavedFunctionId: TypeAlias = (
|
|
1196
|
+
NullableSavedFunctionIdNullableSavedFunctionId | NullableSavedFunctionIdNullableSavedFunctionId1 | None
|
|
1197
|
+
)
|
|
1198
|
+
"""
|
|
1199
|
+
Default preprocessor for this project. When set, functions that use preprocessors will use this instead of their built-in default.
|
|
1200
|
+
"""
|
|
1062
1201
|
|
|
1063
1202
|
|
|
1064
1203
|
class ObjectReference(TypedDict):
|
|
@@ -1074,11 +1213,11 @@ class ObjectReference(TypedDict):
|
|
|
1074
1213
|
"""
|
|
1075
1214
|
ID of the original event.
|
|
1076
1215
|
"""
|
|
1077
|
-
_xact_id: NotRequired[
|
|
1216
|
+
_xact_id: NotRequired[str | None]
|
|
1078
1217
|
"""
|
|
1079
1218
|
Transaction ID of the original event.
|
|
1080
1219
|
"""
|
|
1081
|
-
created: NotRequired[
|
|
1220
|
+
created: NotRequired[str | None]
|
|
1082
1221
|
"""
|
|
1083
1222
|
Created timestamp of the original event. Used to help sort in the UI
|
|
1084
1223
|
"""
|
|
@@ -1097,11 +1236,11 @@ class ObjectReferenceNullish(TypedDict):
|
|
|
1097
1236
|
"""
|
|
1098
1237
|
ID of the original event.
|
|
1099
1238
|
"""
|
|
1100
|
-
_xact_id: NotRequired[
|
|
1239
|
+
_xact_id: NotRequired[str | None]
|
|
1101
1240
|
"""
|
|
1102
1241
|
Transaction ID of the original event.
|
|
1103
1242
|
"""
|
|
1104
|
-
created: NotRequired[
|
|
1243
|
+
created: NotRequired[str | None]
|
|
1105
1244
|
"""
|
|
1106
1245
|
Created timestamp of the original event. Used to help sort in the UI
|
|
1107
1246
|
"""
|
|
@@ -1116,17 +1255,24 @@ class Organization(TypedDict):
|
|
|
1116
1255
|
"""
|
|
1117
1256
|
Name of the organization
|
|
1118
1257
|
"""
|
|
1119
|
-
api_url: NotRequired[
|
|
1120
|
-
is_universal_api: NotRequired[
|
|
1121
|
-
proxy_url: NotRequired[
|
|
1122
|
-
realtime_url: NotRequired[
|
|
1123
|
-
created: NotRequired[
|
|
1258
|
+
api_url: NotRequired[str | None]
|
|
1259
|
+
is_universal_api: NotRequired[bool | None]
|
|
1260
|
+
proxy_url: NotRequired[str | None]
|
|
1261
|
+
realtime_url: NotRequired[str | None]
|
|
1262
|
+
created: NotRequired[str | None]
|
|
1124
1263
|
"""
|
|
1125
1264
|
Date of organization creation
|
|
1126
1265
|
"""
|
|
1127
1266
|
|
|
1128
1267
|
|
|
1129
|
-
Permission = Literal[
|
|
1268
|
+
Permission: TypeAlias = Literal[
|
|
1269
|
+
'create', 'read', 'update', 'delete', 'create_acls', 'read_acls', 'update_acls', 'delete_acls'
|
|
1270
|
+
]
|
|
1271
|
+
"""
|
|
1272
|
+
Each permission permits a certain type of operation on an object in the system
|
|
1273
|
+
|
|
1274
|
+
Permissions can be assigned to to objects on an individual basis, or grouped into roles
|
|
1275
|
+
"""
|
|
1130
1276
|
|
|
1131
1277
|
|
|
1132
1278
|
class ProjectAutomationConfigAction(TypedDict):
|
|
@@ -1140,6 +1286,25 @@ class ProjectAutomationConfigAction(TypedDict):
|
|
|
1140
1286
|
"""
|
|
1141
1287
|
|
|
1142
1288
|
|
|
1289
|
+
class ProjectAutomationConfigAction1(TypedDict):
|
|
1290
|
+
type: Literal['slack']
|
|
1291
|
+
"""
|
|
1292
|
+
The type of action to take
|
|
1293
|
+
"""
|
|
1294
|
+
workspace_id: str
|
|
1295
|
+
"""
|
|
1296
|
+
The Slack workspace ID to post to
|
|
1297
|
+
"""
|
|
1298
|
+
channel: str
|
|
1299
|
+
"""
|
|
1300
|
+
The Slack channel ID to post to
|
|
1301
|
+
"""
|
|
1302
|
+
message_template: NotRequired[str | None]
|
|
1303
|
+
"""
|
|
1304
|
+
Custom message template for the alert
|
|
1305
|
+
"""
|
|
1306
|
+
|
|
1307
|
+
|
|
1143
1308
|
class ProjectAutomationConfig(TypedDict):
|
|
1144
1309
|
event_type: Literal['logs']
|
|
1145
1310
|
"""
|
|
@@ -1153,7 +1318,7 @@ class ProjectAutomationConfig(TypedDict):
|
|
|
1153
1318
|
"""
|
|
1154
1319
|
Perform the triggered action at most once in this interval of seconds
|
|
1155
1320
|
"""
|
|
1156
|
-
action: ProjectAutomationConfigAction
|
|
1321
|
+
action: ProjectAutomationConfigAction | ProjectAutomationConfigAction1
|
|
1157
1322
|
"""
|
|
1158
1323
|
The action to take when the automation rule is triggered
|
|
1159
1324
|
"""
|
|
@@ -1192,11 +1357,11 @@ class ProjectAutomationConfig1(TypedDict):
|
|
|
1192
1357
|
"""
|
|
1193
1358
|
The type of automation.
|
|
1194
1359
|
"""
|
|
1195
|
-
export_definition:
|
|
1196
|
-
ProjectAutomationConfig1ExportDefinition
|
|
1197
|
-
ProjectAutomationConfig1ExportDefinition1
|
|
1198
|
-
ProjectAutomationConfig1ExportDefinition2
|
|
1199
|
-
|
|
1360
|
+
export_definition: (
|
|
1361
|
+
ProjectAutomationConfig1ExportDefinition
|
|
1362
|
+
| ProjectAutomationConfig1ExportDefinition1
|
|
1363
|
+
| ProjectAutomationConfig1ExportDefinition2
|
|
1364
|
+
)
|
|
1200
1365
|
"""
|
|
1201
1366
|
The definition of what to export
|
|
1202
1367
|
"""
|
|
@@ -1213,64 +1378,109 @@ class ProjectAutomationConfig1(TypedDict):
|
|
|
1213
1378
|
Perform the triggered action at most once in this interval of seconds
|
|
1214
1379
|
"""
|
|
1215
1380
|
credentials: ProjectAutomationConfig1Credentials
|
|
1216
|
-
batch_size: NotRequired[
|
|
1381
|
+
batch_size: NotRequired[float | None]
|
|
1217
1382
|
"""
|
|
1218
1383
|
The number of rows to export in each batch
|
|
1219
1384
|
"""
|
|
1220
1385
|
|
|
1221
1386
|
|
|
1387
|
+
class ProjectAutomationConfig3Action(TypedDict):
|
|
1388
|
+
type: Literal['webhook']
|
|
1389
|
+
"""
|
|
1390
|
+
The type of action to take
|
|
1391
|
+
"""
|
|
1392
|
+
url: str
|
|
1393
|
+
"""
|
|
1394
|
+
The webhook URL to send the request to
|
|
1395
|
+
"""
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
class ProjectAutomationConfig3Action1(TypedDict):
|
|
1399
|
+
type: Literal['slack']
|
|
1400
|
+
"""
|
|
1401
|
+
The type of action to take
|
|
1402
|
+
"""
|
|
1403
|
+
workspace_id: str
|
|
1404
|
+
"""
|
|
1405
|
+
The Slack workspace ID to post to
|
|
1406
|
+
"""
|
|
1407
|
+
channel: str
|
|
1408
|
+
"""
|
|
1409
|
+
The Slack channel ID to post to
|
|
1410
|
+
"""
|
|
1411
|
+
message_template: NotRequired[str | None]
|
|
1412
|
+
"""
|
|
1413
|
+
Custom message template for the alert
|
|
1414
|
+
"""
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
class ProjectAutomationConfig3(TypedDict):
|
|
1418
|
+
event_type: Literal['environment_update']
|
|
1419
|
+
"""
|
|
1420
|
+
The type of automation.
|
|
1421
|
+
"""
|
|
1422
|
+
environment_filter: NotRequired[Sequence[str] | None]
|
|
1423
|
+
"""
|
|
1424
|
+
Optional list of environment slugs to filter by
|
|
1425
|
+
"""
|
|
1426
|
+
action: ProjectAutomationConfig3Action | ProjectAutomationConfig3Action1
|
|
1427
|
+
"""
|
|
1428
|
+
The action to take when the automation rule is triggered
|
|
1429
|
+
"""
|
|
1430
|
+
|
|
1431
|
+
|
|
1222
1432
|
class ProjectLogsEventMetadata(TypedDict):
|
|
1223
|
-
model: NotRequired[
|
|
1433
|
+
model: NotRequired[str | None]
|
|
1224
1434
|
"""
|
|
1225
1435
|
The model used for this example
|
|
1226
1436
|
"""
|
|
1227
1437
|
|
|
1228
1438
|
|
|
1229
1439
|
class ProjectLogsEventMetrics(TypedDict):
|
|
1230
|
-
start: NotRequired[
|
|
1440
|
+
start: NotRequired[float | None]
|
|
1231
1441
|
"""
|
|
1232
1442
|
A unix timestamp recording when the section of code which produced the project logs event started
|
|
1233
1443
|
"""
|
|
1234
|
-
end: NotRequired[
|
|
1444
|
+
end: NotRequired[float | None]
|
|
1235
1445
|
"""
|
|
1236
1446
|
A unix timestamp recording when the section of code which produced the project logs event finished
|
|
1237
1447
|
"""
|
|
1238
|
-
prompt_tokens: NotRequired[
|
|
1448
|
+
prompt_tokens: NotRequired[int | None]
|
|
1239
1449
|
"""
|
|
1240
1450
|
The number of tokens in the prompt used to generate the project logs event (only set if this is an LLM span)
|
|
1241
1451
|
"""
|
|
1242
|
-
completion_tokens: NotRequired[
|
|
1452
|
+
completion_tokens: NotRequired[int | None]
|
|
1243
1453
|
"""
|
|
1244
1454
|
The number of tokens in the completion generated by the model (only set if this is an LLM span)
|
|
1245
1455
|
"""
|
|
1246
|
-
tokens: NotRequired[
|
|
1456
|
+
tokens: NotRequired[int | None]
|
|
1247
1457
|
"""
|
|
1248
1458
|
The total number of tokens in the input and output of the project logs event.
|
|
1249
1459
|
"""
|
|
1250
|
-
caller_functionname: NotRequired[
|
|
1460
|
+
caller_functionname: NotRequired[Any | None]
|
|
1251
1461
|
"""
|
|
1252
1462
|
This metric is deprecated
|
|
1253
1463
|
"""
|
|
1254
|
-
caller_filename: NotRequired[
|
|
1464
|
+
caller_filename: NotRequired[Any | None]
|
|
1255
1465
|
"""
|
|
1256
1466
|
This metric is deprecated
|
|
1257
1467
|
"""
|
|
1258
|
-
caller_lineno: NotRequired[
|
|
1468
|
+
caller_lineno: NotRequired[Any | None]
|
|
1259
1469
|
"""
|
|
1260
1470
|
This metric is deprecated
|
|
1261
1471
|
"""
|
|
1262
1472
|
|
|
1263
1473
|
|
|
1264
1474
|
class ProjectLogsEventContext(TypedDict):
|
|
1265
|
-
caller_functionname: NotRequired[
|
|
1475
|
+
caller_functionname: NotRequired[str | None]
|
|
1266
1476
|
"""
|
|
1267
1477
|
The function in code which created the project logs event
|
|
1268
1478
|
"""
|
|
1269
|
-
caller_filename: NotRequired[
|
|
1479
|
+
caller_filename: NotRequired[str | None]
|
|
1270
1480
|
"""
|
|
1271
1481
|
Name of the file in code where the project logs event was created
|
|
1272
1482
|
"""
|
|
1273
|
-
caller_lineno: NotRequired[
|
|
1483
|
+
caller_lineno: NotRequired[int | None]
|
|
1274
1484
|
"""
|
|
1275
1485
|
Line of code where the project logs event was created
|
|
1276
1486
|
"""
|
|
@@ -1287,43 +1497,47 @@ class ProjectScoreCategory(TypedDict):
|
|
|
1287
1497
|
"""
|
|
1288
1498
|
|
|
1289
1499
|
|
|
1290
|
-
ProjectScoreType = Literal['slider', 'categorical', 'weighted', 'minimum', 'maximum', 'online', 'free-form']
|
|
1500
|
+
ProjectScoreType: TypeAlias = Literal['slider', 'categorical', 'weighted', 'minimum', 'maximum', 'online', 'free-form']
|
|
1501
|
+
"""
|
|
1502
|
+
The type of the configured score
|
|
1503
|
+
"""
|
|
1291
1504
|
|
|
1292
1505
|
|
|
1293
1506
|
class ProjectSettingsSpanFieldOrderItem(TypedDict):
|
|
1294
1507
|
object_type: str
|
|
1295
1508
|
column_id: str
|
|
1296
1509
|
position: str
|
|
1297
|
-
layout: NotRequired[
|
|
1510
|
+
layout: NotRequired[Literal['full'] | Literal['two_column'] | None]
|
|
1298
1511
|
|
|
1299
1512
|
|
|
1300
1513
|
class ProjectSettingsRemoteEvalSource(TypedDict):
|
|
1301
1514
|
url: str
|
|
1302
1515
|
name: str
|
|
1303
|
-
description: NotRequired[
|
|
1516
|
+
description: NotRequired[str | None]
|
|
1304
1517
|
|
|
1305
1518
|
|
|
1306
1519
|
class ProjectSettings(TypedDict):
|
|
1307
|
-
comparison_key: NotRequired[
|
|
1520
|
+
comparison_key: NotRequired[str | None]
|
|
1308
1521
|
"""
|
|
1309
1522
|
The key used to join two experiments (defaults to `input`)
|
|
1310
1523
|
"""
|
|
1311
|
-
baseline_experiment_id: NotRequired[
|
|
1524
|
+
baseline_experiment_id: NotRequired[str | None]
|
|
1312
1525
|
"""
|
|
1313
1526
|
The id of the experiment to use as the default baseline for comparisons
|
|
1314
1527
|
"""
|
|
1315
|
-
spanFieldOrder: NotRequired[
|
|
1528
|
+
spanFieldOrder: NotRequired[Sequence[ProjectSettingsSpanFieldOrderItem] | None]
|
|
1316
1529
|
"""
|
|
1317
1530
|
The order of the fields to display in the trace view
|
|
1318
1531
|
"""
|
|
1319
|
-
remote_eval_sources: NotRequired[
|
|
1532
|
+
remote_eval_sources: NotRequired[Sequence[ProjectSettingsRemoteEvalSource] | None]
|
|
1320
1533
|
"""
|
|
1321
1534
|
The remote eval sources to use for the project
|
|
1322
1535
|
"""
|
|
1323
|
-
disable_realtime_queries: NotRequired[
|
|
1536
|
+
disable_realtime_queries: NotRequired[bool | None]
|
|
1324
1537
|
"""
|
|
1325
1538
|
If true, disable real-time queries for this project. This can improve query performance for high-volume logs.
|
|
1326
1539
|
"""
|
|
1540
|
+
default_preprocessor: NotRequired[NullableSavedFunctionId | None]
|
|
1327
1541
|
|
|
1328
1542
|
|
|
1329
1543
|
class ProjectTag(TypedDict):
|
|
@@ -1336,7 +1550,7 @@ class ProjectTag(TypedDict):
|
|
|
1336
1550
|
Unique identifier for the project that the project tag belongs under
|
|
1337
1551
|
"""
|
|
1338
1552
|
user_id: str
|
|
1339
|
-
created: NotRequired[
|
|
1553
|
+
created: NotRequired[str | None]
|
|
1340
1554
|
"""
|
|
1341
1555
|
Date of project tag creation
|
|
1342
1556
|
"""
|
|
@@ -1344,15 +1558,15 @@ class ProjectTag(TypedDict):
|
|
|
1344
1558
|
"""
|
|
1345
1559
|
Name of the project tag
|
|
1346
1560
|
"""
|
|
1347
|
-
description: NotRequired[
|
|
1561
|
+
description: NotRequired[str | None]
|
|
1348
1562
|
"""
|
|
1349
1563
|
Textual description of the project tag
|
|
1350
1564
|
"""
|
|
1351
|
-
color: NotRequired[
|
|
1565
|
+
color: NotRequired[str | None]
|
|
1352
1566
|
"""
|
|
1353
1567
|
Color of the tag for the UI
|
|
1354
1568
|
"""
|
|
1355
|
-
position: NotRequired[
|
|
1569
|
+
position: NotRequired[str | None]
|
|
1356
1570
|
"""
|
|
1357
1571
|
An optional LexoRank-based string that sets the sort position for the tag in the UI
|
|
1358
1572
|
"""
|
|
@@ -1369,15 +1583,15 @@ class PromptBlockDataNullishPromptBlockDataNullish1(TypedDict):
|
|
|
1369
1583
|
|
|
1370
1584
|
|
|
1371
1585
|
class PromptDataOrigin(TypedDict):
|
|
1372
|
-
prompt_id: NotRequired[
|
|
1373
|
-
project_id: NotRequired[
|
|
1374
|
-
prompt_version: NotRequired[
|
|
1586
|
+
prompt_id: NotRequired[str | None]
|
|
1587
|
+
project_id: NotRequired[str | None]
|
|
1588
|
+
prompt_version: NotRequired[str | None]
|
|
1375
1589
|
|
|
1376
1590
|
|
|
1377
1591
|
class PromptDataNullishOrigin(TypedDict):
|
|
1378
|
-
prompt_id: NotRequired[
|
|
1379
|
-
project_id: NotRequired[
|
|
1380
|
-
prompt_version: NotRequired[
|
|
1592
|
+
prompt_id: NotRequired[str | None]
|
|
1593
|
+
project_id: NotRequired[str | None]
|
|
1594
|
+
prompt_version: NotRequired[str | None]
|
|
1381
1595
|
|
|
1382
1596
|
|
|
1383
1597
|
class PromptParserNullish(TypedDict):
|
|
@@ -1399,7 +1613,7 @@ class PromptSessionEvent(TypedDict):
|
|
|
1399
1613
|
"""
|
|
1400
1614
|
The timestamp the prompt session event was created
|
|
1401
1615
|
"""
|
|
1402
|
-
_pagination_key: NotRequired[
|
|
1616
|
+
_pagination_key: NotRequired[str | None]
|
|
1403
1617
|
"""
|
|
1404
1618
|
A stable, time-ordered key that can be used to paginate over prompt session events. This field is auto-generated by Braintrust and only exists in Brainstore.
|
|
1405
1619
|
"""
|
|
@@ -1411,67 +1625,67 @@ class PromptSessionEvent(TypedDict):
|
|
|
1411
1625
|
"""
|
|
1412
1626
|
Unique identifier for the prompt
|
|
1413
1627
|
"""
|
|
1414
|
-
prompt_session_data: NotRequired[
|
|
1628
|
+
prompt_session_data: NotRequired[Any | None]
|
|
1415
1629
|
"""
|
|
1416
1630
|
Data about the prompt session
|
|
1417
1631
|
"""
|
|
1418
|
-
prompt_data: NotRequired[
|
|
1632
|
+
prompt_data: NotRequired[Any | None]
|
|
1419
1633
|
"""
|
|
1420
1634
|
Data about the prompt
|
|
1421
1635
|
"""
|
|
1422
|
-
function_data: NotRequired[
|
|
1636
|
+
function_data: NotRequired[Any | None]
|
|
1423
1637
|
"""
|
|
1424
1638
|
Data about the function
|
|
1425
1639
|
"""
|
|
1426
|
-
function_type: NotRequired[
|
|
1427
|
-
object_data: NotRequired[
|
|
1640
|
+
function_type: NotRequired[FunctionTypeEnumNullish | None]
|
|
1641
|
+
object_data: NotRequired[Any | None]
|
|
1428
1642
|
"""
|
|
1429
1643
|
Data about the mapped data
|
|
1430
1644
|
"""
|
|
1431
|
-
completion: NotRequired[
|
|
1645
|
+
completion: NotRequired[Any | None]
|
|
1432
1646
|
"""
|
|
1433
1647
|
Data about the completion
|
|
1434
1648
|
"""
|
|
1435
|
-
tags: NotRequired[
|
|
1649
|
+
tags: NotRequired[Sequence[str] | None]
|
|
1436
1650
|
"""
|
|
1437
1651
|
A list of tags to log
|
|
1438
1652
|
"""
|
|
1439
1653
|
|
|
1440
1654
|
|
|
1441
1655
|
class RepoInfo(TypedDict):
|
|
1442
|
-
commit: NotRequired[
|
|
1656
|
+
commit: NotRequired[str | None]
|
|
1443
1657
|
"""
|
|
1444
1658
|
SHA of most recent commit
|
|
1445
1659
|
"""
|
|
1446
|
-
branch: NotRequired[
|
|
1660
|
+
branch: NotRequired[str | None]
|
|
1447
1661
|
"""
|
|
1448
1662
|
Name of the branch the most recent commit belongs to
|
|
1449
1663
|
"""
|
|
1450
|
-
tag: NotRequired[
|
|
1664
|
+
tag: NotRequired[str | None]
|
|
1451
1665
|
"""
|
|
1452
1666
|
Name of the tag on the most recent commit
|
|
1453
1667
|
"""
|
|
1454
|
-
dirty: NotRequired[
|
|
1668
|
+
dirty: NotRequired[bool | None]
|
|
1455
1669
|
"""
|
|
1456
1670
|
Whether or not the repo had uncommitted changes when snapshotted
|
|
1457
1671
|
"""
|
|
1458
|
-
author_name: NotRequired[
|
|
1672
|
+
author_name: NotRequired[str | None]
|
|
1459
1673
|
"""
|
|
1460
1674
|
Name of the author of the most recent commit
|
|
1461
1675
|
"""
|
|
1462
|
-
author_email: NotRequired[
|
|
1676
|
+
author_email: NotRequired[str | None]
|
|
1463
1677
|
"""
|
|
1464
1678
|
Email of the author of the most recent commit
|
|
1465
1679
|
"""
|
|
1466
|
-
commit_message: NotRequired[
|
|
1680
|
+
commit_message: NotRequired[str | None]
|
|
1467
1681
|
"""
|
|
1468
1682
|
Most recent commit message
|
|
1469
1683
|
"""
|
|
1470
|
-
commit_time: NotRequired[
|
|
1684
|
+
commit_time: NotRequired[str | None]
|
|
1471
1685
|
"""
|
|
1472
1686
|
Time of the most recent commit
|
|
1473
1687
|
"""
|
|
1474
|
-
git_diff: NotRequired[
|
|
1688
|
+
git_diff: NotRequired[str | None]
|
|
1475
1689
|
"""
|
|
1476
1690
|
If the repo was dirty when run, this includes the diff between the current state of the repo and the most recent commit.
|
|
1477
1691
|
"""
|
|
@@ -1487,9 +1701,9 @@ class ResponseFormatResponseFormat2(TypedDict):
|
|
|
1487
1701
|
|
|
1488
1702
|
class ResponseFormatJsonSchema(TypedDict):
|
|
1489
1703
|
name: str
|
|
1490
|
-
description: NotRequired[
|
|
1491
|
-
schema: NotRequired[
|
|
1492
|
-
strict: NotRequired[
|
|
1704
|
+
description: NotRequired[str | None]
|
|
1705
|
+
schema: NotRequired[Mapping[str, Any] | str | None]
|
|
1706
|
+
strict: NotRequired[bool | None]
|
|
1493
1707
|
|
|
1494
1708
|
|
|
1495
1709
|
class ResponseFormatNullishResponseFormatNullish(TypedDict):
|
|
@@ -1505,21 +1719,23 @@ class ResponseFormatNullishResponseFormatNullish2(TypedDict):
|
|
|
1505
1719
|
type: Literal['text']
|
|
1506
1720
|
|
|
1507
1721
|
|
|
1508
|
-
ResponseFormatNullish =
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
]
|
|
1722
|
+
ResponseFormatNullish: TypeAlias = (
|
|
1723
|
+
ResponseFormatNullishResponseFormatNullish
|
|
1724
|
+
| ResponseFormatNullishResponseFormatNullish1
|
|
1725
|
+
| ResponseFormatNullishResponseFormatNullish2
|
|
1726
|
+
| None
|
|
1727
|
+
)
|
|
1515
1728
|
|
|
1516
1729
|
|
|
1517
|
-
RetentionObjectType = Literal['project_logs', 'experiment', 'dataset']
|
|
1730
|
+
RetentionObjectType: TypeAlias = Literal['project_logs', 'experiment', 'dataset']
|
|
1731
|
+
"""
|
|
1732
|
+
The object type that the retention policy applies to
|
|
1733
|
+
"""
|
|
1518
1734
|
|
|
1519
1735
|
|
|
1520
1736
|
class RoleMemberPermission(TypedDict):
|
|
1521
1737
|
permission: Permission
|
|
1522
|
-
restrict_object_type: NotRequired[
|
|
1738
|
+
restrict_object_type: NotRequired[AclObjectType | None]
|
|
1523
1739
|
|
|
1524
1740
|
|
|
1525
1741
|
class Role(TypedDict):
|
|
@@ -1527,7 +1743,7 @@ class Role(TypedDict):
|
|
|
1527
1743
|
"""
|
|
1528
1744
|
Unique identifier for the role
|
|
1529
1745
|
"""
|
|
1530
|
-
org_id: NotRequired[
|
|
1746
|
+
org_id: NotRequired[str | None]
|
|
1531
1747
|
"""
|
|
1532
1748
|
Unique id for the organization that the role belongs under
|
|
1533
1749
|
|
|
@@ -1535,11 +1751,11 @@ class Role(TypedDict):
|
|
|
1535
1751
|
|
|
1536
1752
|
It is forbidden to change the org after creating a role
|
|
1537
1753
|
"""
|
|
1538
|
-
user_id: NotRequired[
|
|
1754
|
+
user_id: NotRequired[str | None]
|
|
1539
1755
|
"""
|
|
1540
1756
|
Identifies the user who created the role
|
|
1541
1757
|
"""
|
|
1542
|
-
created: NotRequired[
|
|
1758
|
+
created: NotRequired[str | None]
|
|
1543
1759
|
"""
|
|
1544
1760
|
Date of role creation
|
|
1545
1761
|
"""
|
|
@@ -1547,19 +1763,19 @@ class Role(TypedDict):
|
|
|
1547
1763
|
"""
|
|
1548
1764
|
Name of the role
|
|
1549
1765
|
"""
|
|
1550
|
-
description: NotRequired[
|
|
1766
|
+
description: NotRequired[str | None]
|
|
1551
1767
|
"""
|
|
1552
1768
|
Textual description of the role
|
|
1553
1769
|
"""
|
|
1554
|
-
deleted_at: NotRequired[
|
|
1770
|
+
deleted_at: NotRequired[str | None]
|
|
1555
1771
|
"""
|
|
1556
1772
|
Date of role deletion, or null if the role is still active
|
|
1557
1773
|
"""
|
|
1558
|
-
member_permissions: NotRequired[
|
|
1774
|
+
member_permissions: NotRequired[Sequence[RoleMemberPermission] | None]
|
|
1559
1775
|
"""
|
|
1560
1776
|
(permission, restrict_object_type) tuples which belong to this role
|
|
1561
1777
|
"""
|
|
1562
|
-
member_roles: NotRequired[
|
|
1778
|
+
member_roles: NotRequired[Sequence[str] | None]
|
|
1563
1779
|
"""
|
|
1564
1780
|
Ids of the roles this role inherits from
|
|
1565
1781
|
|
|
@@ -1569,17 +1785,156 @@ class Role(TypedDict):
|
|
|
1569
1785
|
|
|
1570
1786
|
class RunEvalData(TypedDict):
|
|
1571
1787
|
dataset_id: str
|
|
1572
|
-
_internal_btql: NotRequired[
|
|
1788
|
+
_internal_btql: NotRequired[Mapping[str, Any] | None]
|
|
1573
1789
|
|
|
1574
1790
|
|
|
1575
1791
|
class RunEvalData1(TypedDict):
|
|
1576
1792
|
project_name: str
|
|
1577
1793
|
dataset_name: str
|
|
1578
|
-
_internal_btql: NotRequired[
|
|
1794
|
+
_internal_btql: NotRequired[Mapping[str, Any] | None]
|
|
1579
1795
|
|
|
1580
1796
|
|
|
1581
1797
|
class RunEvalData2(TypedDict):
|
|
1582
|
-
data: Sequence
|
|
1798
|
+
data: Sequence[Any]
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
class TaskTask(TypedDict):
|
|
1802
|
+
function_id: str
|
|
1803
|
+
"""
|
|
1804
|
+
The ID of the function
|
|
1805
|
+
"""
|
|
1806
|
+
version: NotRequired[str | None]
|
|
1807
|
+
"""
|
|
1808
|
+
The version of the function
|
|
1809
|
+
"""
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
class TaskTask1(TypedDict):
|
|
1813
|
+
project_name: str
|
|
1814
|
+
"""
|
|
1815
|
+
The name of the project containing the function
|
|
1816
|
+
"""
|
|
1817
|
+
slug: str
|
|
1818
|
+
"""
|
|
1819
|
+
The slug of the function
|
|
1820
|
+
"""
|
|
1821
|
+
version: NotRequired[str | None]
|
|
1822
|
+
"""
|
|
1823
|
+
The version of the function
|
|
1824
|
+
"""
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
class TaskTask2(TypedDict):
|
|
1828
|
+
global_function: str
|
|
1829
|
+
"""
|
|
1830
|
+
The name of the global function. Currently, the global namespace includes the functions in autoevals
|
|
1831
|
+
"""
|
|
1832
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
class TaskTask3(TypedDict):
|
|
1836
|
+
prompt_session_id: str
|
|
1837
|
+
"""
|
|
1838
|
+
The ID of the prompt session
|
|
1839
|
+
"""
|
|
1840
|
+
prompt_session_function_id: str
|
|
1841
|
+
"""
|
|
1842
|
+
The ID of the function in the prompt session
|
|
1843
|
+
"""
|
|
1844
|
+
version: NotRequired[str | None]
|
|
1845
|
+
"""
|
|
1846
|
+
The version of the function
|
|
1847
|
+
"""
|
|
1848
|
+
|
|
1849
|
+
|
|
1850
|
+
class TaskTask4InlineContext(TypedDict):
|
|
1851
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
1852
|
+
version: str
|
|
1853
|
+
|
|
1854
|
+
|
|
1855
|
+
class TaskTask4(TypedDict):
|
|
1856
|
+
inline_context: TaskTask4InlineContext
|
|
1857
|
+
code: str
|
|
1858
|
+
"""
|
|
1859
|
+
The inline code to execute
|
|
1860
|
+
"""
|
|
1861
|
+
name: NotRequired[str | None]
|
|
1862
|
+
"""
|
|
1863
|
+
The name of the inline code function
|
|
1864
|
+
"""
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
class TaskTask7(TypedDict):
|
|
1868
|
+
pass
|
|
1869
|
+
|
|
1870
|
+
|
|
1871
|
+
class TaskTask8(TaskTask, TaskTask7):
|
|
1872
|
+
pass
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
class TaskTask9(TaskTask1, TaskTask7):
|
|
1876
|
+
pass
|
|
1877
|
+
|
|
1878
|
+
|
|
1879
|
+
class TaskTask10(TaskTask2, TaskTask7):
|
|
1880
|
+
pass
|
|
1881
|
+
|
|
1882
|
+
|
|
1883
|
+
class TaskTask11(TaskTask3, TaskTask7):
|
|
1884
|
+
pass
|
|
1885
|
+
|
|
1886
|
+
|
|
1887
|
+
class TaskTask12(TaskTask4, TaskTask7):
|
|
1888
|
+
pass
|
|
1889
|
+
|
|
1890
|
+
|
|
1891
|
+
class ParentParentRowIds(TypedDict):
|
|
1892
|
+
id: str
|
|
1893
|
+
"""
|
|
1894
|
+
The id of the row
|
|
1895
|
+
"""
|
|
1896
|
+
span_id: str
|
|
1897
|
+
"""
|
|
1898
|
+
The span_id of the row
|
|
1899
|
+
"""
|
|
1900
|
+
root_span_id: str
|
|
1901
|
+
"""
|
|
1902
|
+
The root_span_id of the row
|
|
1903
|
+
"""
|
|
1904
|
+
|
|
1905
|
+
|
|
1906
|
+
class ParentParent(TypedDict):
|
|
1907
|
+
object_type: Literal['project_logs', 'experiment', 'playground_logs']
|
|
1908
|
+
object_id: str
|
|
1909
|
+
"""
|
|
1910
|
+
The id of the container object you are logging to
|
|
1911
|
+
"""
|
|
1912
|
+
row_ids: NotRequired[ParentParentRowIds | None]
|
|
1913
|
+
"""
|
|
1914
|
+
Identifiers for the row to to log a subspan under
|
|
1915
|
+
"""
|
|
1916
|
+
propagated_event: NotRequired[Mapping[str, Any] | None]
|
|
1917
|
+
"""
|
|
1918
|
+
Include these properties in every span created under this parent
|
|
1919
|
+
"""
|
|
1920
|
+
|
|
1921
|
+
|
|
1922
|
+
class ParentParent1(TypedDict):
|
|
1923
|
+
pass
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
class ParentParent2(ParentParent, ParentParent1):
|
|
1927
|
+
pass
|
|
1928
|
+
|
|
1929
|
+
|
|
1930
|
+
Parent: TypeAlias = ParentParent2
|
|
1931
|
+
|
|
1932
|
+
|
|
1933
|
+
class RunEvalMcpAuth(TypedDict):
|
|
1934
|
+
oauth_token: NotRequired[str | None]
|
|
1935
|
+
"""
|
|
1936
|
+
The OAuth token to use
|
|
1937
|
+
"""
|
|
1583
1938
|
|
|
1584
1939
|
|
|
1585
1940
|
class SavedFunctionIdSavedFunctionId(TypedDict):
|
|
@@ -1590,9 +1945,10 @@ class SavedFunctionIdSavedFunctionId(TypedDict):
|
|
|
1590
1945
|
class SavedFunctionIdSavedFunctionId1(TypedDict):
|
|
1591
1946
|
type: Literal['global']
|
|
1592
1947
|
name: str
|
|
1948
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
1593
1949
|
|
|
1594
1950
|
|
|
1595
|
-
SavedFunctionId =
|
|
1951
|
+
SavedFunctionId: TypeAlias = SavedFunctionIdSavedFunctionId | SavedFunctionIdSavedFunctionId1
|
|
1596
1952
|
|
|
1597
1953
|
|
|
1598
1954
|
class ServiceToken(TypedDict):
|
|
@@ -1600,7 +1956,7 @@ class ServiceToken(TypedDict):
|
|
|
1600
1956
|
"""
|
|
1601
1957
|
Unique identifier for the service token
|
|
1602
1958
|
"""
|
|
1603
|
-
created: NotRequired[
|
|
1959
|
+
created: NotRequired[str | None]
|
|
1604
1960
|
"""
|
|
1605
1961
|
Date of service token creation
|
|
1606
1962
|
"""
|
|
@@ -1609,19 +1965,19 @@ class ServiceToken(TypedDict):
|
|
|
1609
1965
|
Name of the service token
|
|
1610
1966
|
"""
|
|
1611
1967
|
preview_name: str
|
|
1612
|
-
service_account_id: NotRequired[
|
|
1968
|
+
service_account_id: NotRequired[str | None]
|
|
1613
1969
|
"""
|
|
1614
1970
|
Unique identifier for the service token
|
|
1615
1971
|
"""
|
|
1616
|
-
service_account_email: NotRequired[
|
|
1972
|
+
service_account_email: NotRequired[str | None]
|
|
1617
1973
|
"""
|
|
1618
1974
|
The service account email (not routable)
|
|
1619
1975
|
"""
|
|
1620
|
-
service_account_name: NotRequired[
|
|
1976
|
+
service_account_name: NotRequired[str | None]
|
|
1621
1977
|
"""
|
|
1622
1978
|
The service account name
|
|
1623
1979
|
"""
|
|
1624
|
-
org_id: NotRequired[
|
|
1980
|
+
org_id: NotRequired[str | None]
|
|
1625
1981
|
"""
|
|
1626
1982
|
Unique identifier for the organization
|
|
1627
1983
|
"""
|
|
@@ -1636,15 +1992,15 @@ class SpanIFrame(TypedDict):
|
|
|
1636
1992
|
"""
|
|
1637
1993
|
Unique identifier for the project that the span iframe belongs under
|
|
1638
1994
|
"""
|
|
1639
|
-
user_id: NotRequired[
|
|
1995
|
+
user_id: NotRequired[str | None]
|
|
1640
1996
|
"""
|
|
1641
1997
|
Identifies the user who created the span iframe
|
|
1642
1998
|
"""
|
|
1643
|
-
created: NotRequired[
|
|
1999
|
+
created: NotRequired[str | None]
|
|
1644
2000
|
"""
|
|
1645
2001
|
Date of span iframe creation
|
|
1646
2002
|
"""
|
|
1647
|
-
deleted_at: NotRequired[
|
|
2003
|
+
deleted_at: NotRequired[str | None]
|
|
1648
2004
|
"""
|
|
1649
2005
|
Date of span iframe deletion, or null if the span iframe is still active
|
|
1650
2006
|
"""
|
|
@@ -1652,7 +2008,7 @@ class SpanIFrame(TypedDict):
|
|
|
1652
2008
|
"""
|
|
1653
2009
|
Name of the span iframe
|
|
1654
2010
|
"""
|
|
1655
|
-
description: NotRequired[
|
|
2011
|
+
description: NotRequired[str | None]
|
|
1656
2012
|
"""
|
|
1657
2013
|
Textual description of the span iframe
|
|
1658
2014
|
"""
|
|
@@ -1660,13 +2016,28 @@ class SpanIFrame(TypedDict):
|
|
|
1660
2016
|
"""
|
|
1661
2017
|
URL to embed the project viewer in an iframe
|
|
1662
2018
|
"""
|
|
1663
|
-
post_message: NotRequired[
|
|
2019
|
+
post_message: NotRequired[bool | None]
|
|
1664
2020
|
"""
|
|
1665
2021
|
Whether to post messages to the iframe containing the span's data. This is useful when you want to render more data than fits in the URL.
|
|
1666
2022
|
"""
|
|
1667
2023
|
|
|
1668
2024
|
|
|
1669
|
-
|
|
2025
|
+
class SpanScope(TypedDict):
|
|
2026
|
+
type: Literal['span']
|
|
2027
|
+
root_span_id: str
|
|
2028
|
+
"""
|
|
2029
|
+
The root span id is a unique identifier for the trace.
|
|
2030
|
+
"""
|
|
2031
|
+
id: str
|
|
2032
|
+
"""
|
|
2033
|
+
A unique identifier for the span.
|
|
2034
|
+
"""
|
|
2035
|
+
|
|
2036
|
+
|
|
2037
|
+
SpanType: TypeAlias = Literal['llm', 'score', 'function', 'eval', 'task', 'tool']
|
|
2038
|
+
"""
|
|
2039
|
+
Type of the span, for display purposes only
|
|
2040
|
+
"""
|
|
1670
2041
|
|
|
1671
2042
|
|
|
1672
2043
|
class SSEConsoleEventData(TypedDict):
|
|
@@ -1680,7 +2051,7 @@ class SSEProgressEventData(TypedDict):
|
|
|
1680
2051
|
The id of the span this event is for
|
|
1681
2052
|
"""
|
|
1682
2053
|
object_type: FunctionObjectType
|
|
1683
|
-
origin: NotRequired[
|
|
2054
|
+
origin: NotRequired[ObjectReferenceNullish | None]
|
|
1684
2055
|
format: FunctionFormat
|
|
1685
2056
|
output_type: FunctionOutputType
|
|
1686
2057
|
name: str
|
|
@@ -1688,14 +2059,17 @@ class SSEProgressEventData(TypedDict):
|
|
|
1688
2059
|
data: str
|
|
1689
2060
|
|
|
1690
2061
|
|
|
1691
|
-
StreamingMode = Literal['auto', 'parallel']
|
|
2062
|
+
StreamingMode: TypeAlias = Literal['auto', 'parallel', 'json', 'text']
|
|
2063
|
+
"""
|
|
2064
|
+
The mode format of the returned value (defaults to 'auto')
|
|
2065
|
+
"""
|
|
1692
2066
|
|
|
1693
2067
|
|
|
1694
2068
|
class ToolFunctionDefinitionFunction(TypedDict):
|
|
1695
2069
|
name: str
|
|
1696
|
-
description: NotRequired[
|
|
1697
|
-
parameters: NotRequired[
|
|
1698
|
-
strict: NotRequired[
|
|
2070
|
+
description: NotRequired[str | None]
|
|
2071
|
+
parameters: NotRequired[Mapping[str, Any] | None]
|
|
2072
|
+
strict: NotRequired[bool | None]
|
|
1699
2073
|
|
|
1700
2074
|
|
|
1701
2075
|
class ToolFunctionDefinition(TypedDict):
|
|
@@ -1703,7 +2077,15 @@ class ToolFunctionDefinition(TypedDict):
|
|
|
1703
2077
|
function: ToolFunctionDefinitionFunction
|
|
1704
2078
|
|
|
1705
2079
|
|
|
1706
|
-
|
|
2080
|
+
class TraceScope(TypedDict):
|
|
2081
|
+
type: Literal['trace']
|
|
2082
|
+
root_span_id: str
|
|
2083
|
+
"""
|
|
2084
|
+
The root span id is a unique identifier for the trace.
|
|
2085
|
+
"""
|
|
2086
|
+
|
|
2087
|
+
|
|
2088
|
+
UploadStatus: TypeAlias = Literal['uploading', 'done', 'error']
|
|
1707
2089
|
|
|
1708
2090
|
|
|
1709
2091
|
class User(TypedDict):
|
|
@@ -1711,50 +2093,51 @@ class User(TypedDict):
|
|
|
1711
2093
|
"""
|
|
1712
2094
|
Unique identifier for the user
|
|
1713
2095
|
"""
|
|
1714
|
-
given_name: NotRequired[
|
|
2096
|
+
given_name: NotRequired[str | None]
|
|
1715
2097
|
"""
|
|
1716
2098
|
Given name of the user
|
|
1717
2099
|
"""
|
|
1718
|
-
family_name: NotRequired[
|
|
2100
|
+
family_name: NotRequired[str | None]
|
|
1719
2101
|
"""
|
|
1720
2102
|
Family name of the user
|
|
1721
2103
|
"""
|
|
1722
|
-
email: NotRequired[
|
|
2104
|
+
email: NotRequired[str | None]
|
|
1723
2105
|
"""
|
|
1724
2106
|
The user's email
|
|
1725
2107
|
"""
|
|
1726
|
-
avatar_url: NotRequired[
|
|
2108
|
+
avatar_url: NotRequired[str | None]
|
|
1727
2109
|
"""
|
|
1728
2110
|
URL of the user's Avatar image
|
|
1729
2111
|
"""
|
|
1730
|
-
created: NotRequired[
|
|
2112
|
+
created: NotRequired[str | None]
|
|
1731
2113
|
"""
|
|
1732
2114
|
Date of user creation
|
|
1733
2115
|
"""
|
|
1734
2116
|
|
|
1735
2117
|
|
|
1736
2118
|
class ViewDataSearch(TypedDict):
|
|
1737
|
-
filter: NotRequired[
|
|
1738
|
-
tag: NotRequired[
|
|
1739
|
-
match: NotRequired[
|
|
1740
|
-
sort: NotRequired[
|
|
2119
|
+
filter: NotRequired[Sequence[Any] | None]
|
|
2120
|
+
tag: NotRequired[Sequence[Any] | None]
|
|
2121
|
+
match: NotRequired[Sequence[Any] | None]
|
|
2122
|
+
sort: NotRequired[Sequence[Any] | None]
|
|
1741
2123
|
|
|
1742
2124
|
|
|
1743
2125
|
class ViewOptionsViewOptionsOptions(TypedDict):
|
|
1744
|
-
spanType: NotRequired[
|
|
1745
|
-
rangeValue: NotRequired[
|
|
1746
|
-
frameStart: NotRequired[
|
|
1747
|
-
frameEnd: NotRequired[
|
|
1748
|
-
tzUTC: NotRequired[
|
|
1749
|
-
chartVisibility: NotRequired[
|
|
1750
|
-
projectId: NotRequired[
|
|
1751
|
-
type: NotRequired[
|
|
1752
|
-
groupBy: NotRequired[
|
|
2126
|
+
spanType: NotRequired[Literal['range', 'frame'] | None]
|
|
2127
|
+
rangeValue: NotRequired[str | None]
|
|
2128
|
+
frameStart: NotRequired[str | None]
|
|
2129
|
+
frameEnd: NotRequired[str | None]
|
|
2130
|
+
tzUTC: NotRequired[bool | None]
|
|
2131
|
+
chartVisibility: NotRequired[Mapping[str, Any] | None]
|
|
2132
|
+
projectId: NotRequired[str | None]
|
|
2133
|
+
type: NotRequired[Literal['project', 'experiment'] | None]
|
|
2134
|
+
groupBy: NotRequired[str | None]
|
|
1753
2135
|
|
|
1754
2136
|
|
|
1755
2137
|
class ViewOptionsViewOptions(TypedDict):
|
|
1756
2138
|
viewType: Literal['monitor']
|
|
1757
2139
|
options: ViewOptionsViewOptionsOptions
|
|
2140
|
+
freezeColumns: NotRequired[bool | None]
|
|
1758
2141
|
|
|
1759
2142
|
|
|
1760
2143
|
class ViewOptionsViewOptions1ExcludedMeasure(TypedDict):
|
|
@@ -1792,28 +2175,32 @@ ViewOptionsViewOptions1TimeRangeFilter = TypedDict(
|
|
|
1792
2175
|
|
|
1793
2176
|
|
|
1794
2177
|
class ViewOptionsViewOptions1(TypedDict):
|
|
1795
|
-
columnVisibility: NotRequired[
|
|
1796
|
-
columnOrder: NotRequired[
|
|
1797
|
-
columnSizing: NotRequired[
|
|
1798
|
-
grouping: NotRequired[
|
|
1799
|
-
rowHeight: NotRequired[
|
|
1800
|
-
tallGroupRows: NotRequired[
|
|
1801
|
-
layout: NotRequired[
|
|
1802
|
-
chartHeight: NotRequired[
|
|
1803
|
-
excludedMeasures: NotRequired[
|
|
1804
|
-
yMetric: NotRequired[
|
|
1805
|
-
xAxis: NotRequired[
|
|
1806
|
-
symbolGrouping: NotRequired[
|
|
1807
|
-
xAxisAggregation: NotRequired[
|
|
2178
|
+
columnVisibility: NotRequired[Mapping[str, Any] | None]
|
|
2179
|
+
columnOrder: NotRequired[Sequence[str] | None]
|
|
2180
|
+
columnSizing: NotRequired[Mapping[str, Any] | None]
|
|
2181
|
+
grouping: NotRequired[str | None]
|
|
2182
|
+
rowHeight: NotRequired[str | None]
|
|
2183
|
+
tallGroupRows: NotRequired[bool | None]
|
|
2184
|
+
layout: NotRequired[str | None]
|
|
2185
|
+
chartHeight: NotRequired[float | None]
|
|
2186
|
+
excludedMeasures: NotRequired[Sequence[ViewOptionsViewOptions1ExcludedMeasure] | None]
|
|
2187
|
+
yMetric: NotRequired[ViewOptionsViewOptions1YMetric | None]
|
|
2188
|
+
xAxis: NotRequired[ViewOptionsViewOptions1XAxis | None]
|
|
2189
|
+
symbolGrouping: NotRequired[ViewOptionsViewOptions1SymbolGrouping | None]
|
|
2190
|
+
xAxisAggregation: NotRequired[str | None]
|
|
1808
2191
|
"""
|
|
1809
2192
|
One of 'avg', 'sum', 'min', 'max', 'median', 'all'
|
|
1810
2193
|
"""
|
|
1811
|
-
chartAnnotations: NotRequired[
|
|
1812
|
-
timeRangeFilter: NotRequired[
|
|
1813
|
-
queryShape: NotRequired[
|
|
2194
|
+
chartAnnotations: NotRequired[Sequence[ViewOptionsViewOptions1ChartAnnotation] | None]
|
|
2195
|
+
timeRangeFilter: NotRequired[str | ViewOptionsViewOptions1TimeRangeFilter | None]
|
|
2196
|
+
queryShape: NotRequired[Literal['traces', 'spans'] | None]
|
|
2197
|
+
freezeColumns: NotRequired[bool | None]
|
|
1814
2198
|
|
|
1815
2199
|
|
|
1816
|
-
ViewOptions =
|
|
2200
|
+
ViewOptions: TypeAlias = ViewOptionsViewOptions | ViewOptionsViewOptions1 | None
|
|
2201
|
+
"""
|
|
2202
|
+
Options for the view in the app
|
|
2203
|
+
"""
|
|
1817
2204
|
|
|
1818
2205
|
|
|
1819
2206
|
class Acl(TypedDict):
|
|
@@ -1821,22 +2208,22 @@ class Acl(TypedDict):
|
|
|
1821
2208
|
"""
|
|
1822
2209
|
Unique identifier for the acl
|
|
1823
2210
|
"""
|
|
1824
|
-
object_type:
|
|
2211
|
+
object_type: AclObjectType
|
|
1825
2212
|
object_id: str
|
|
1826
2213
|
"""
|
|
1827
2214
|
The id of the object the ACL applies to
|
|
1828
2215
|
"""
|
|
1829
|
-
user_id: NotRequired[
|
|
2216
|
+
user_id: NotRequired[str | None]
|
|
1830
2217
|
"""
|
|
1831
2218
|
Id of the user the ACL applies to. Exactly one of `user_id` and `group_id` will be provided
|
|
1832
2219
|
"""
|
|
1833
|
-
group_id: NotRequired[
|
|
2220
|
+
group_id: NotRequired[str | None]
|
|
1834
2221
|
"""
|
|
1835
2222
|
Id of the group the ACL applies to. Exactly one of `user_id` and `group_id` will be provided
|
|
1836
2223
|
"""
|
|
1837
|
-
permission: NotRequired[
|
|
1838
|
-
restrict_object_type: NotRequired[
|
|
1839
|
-
role_id: NotRequired[
|
|
2224
|
+
permission: NotRequired[Permission | None]
|
|
2225
|
+
restrict_object_type: NotRequired[AclObjectType | None]
|
|
2226
|
+
role_id: NotRequired[str | None]
|
|
1840
2227
|
"""
|
|
1841
2228
|
Id of the role the ACL grants. Exactly one of `permission` and `role_id` will be provided
|
|
1842
2229
|
"""
|
|
@@ -1844,41 +2231,41 @@ class Acl(TypedDict):
|
|
|
1844
2231
|
"""
|
|
1845
2232
|
The organization the ACL's referred object belongs to
|
|
1846
2233
|
"""
|
|
1847
|
-
created: NotRequired[
|
|
2234
|
+
created: NotRequired[str | None]
|
|
1848
2235
|
"""
|
|
1849
2236
|
Date of acl creation
|
|
1850
2237
|
"""
|
|
1851
2238
|
|
|
1852
2239
|
|
|
1853
2240
|
class AnyModelParams(TypedDict):
|
|
1854
|
-
temperature: NotRequired[
|
|
1855
|
-
top_p: NotRequired[
|
|
2241
|
+
temperature: NotRequired[float | None]
|
|
2242
|
+
top_p: NotRequired[float | None]
|
|
1856
2243
|
max_tokens: float
|
|
1857
|
-
max_completion_tokens: NotRequired[
|
|
2244
|
+
max_completion_tokens: NotRequired[float | None]
|
|
1858
2245
|
"""
|
|
1859
2246
|
The successor to max_tokens
|
|
1860
2247
|
"""
|
|
1861
|
-
frequency_penalty: NotRequired[
|
|
1862
|
-
presence_penalty: NotRequired[
|
|
1863
|
-
response_format: NotRequired[
|
|
1864
|
-
tool_choice: NotRequired[
|
|
1865
|
-
function_call: NotRequired[
|
|
1866
|
-
n: NotRequired[
|
|
1867
|
-
stop: NotRequired[
|
|
1868
|
-
reasoning_effort: NotRequired[
|
|
1869
|
-
verbosity: NotRequired[
|
|
1870
|
-
top_k: NotRequired[
|
|
1871
|
-
stop_sequences: NotRequired[
|
|
1872
|
-
reasoning_enabled: NotRequired[
|
|
1873
|
-
reasoning_budget: NotRequired[
|
|
1874
|
-
max_tokens_to_sample: NotRequired[
|
|
2248
|
+
frequency_penalty: NotRequired[float | None]
|
|
2249
|
+
presence_penalty: NotRequired[float | None]
|
|
2250
|
+
response_format: NotRequired[ResponseFormatNullish | None]
|
|
2251
|
+
tool_choice: NotRequired[Literal['auto'] | Literal['none'] | Literal['required'] | AnyModelParamsToolChoice | None]
|
|
2252
|
+
function_call: NotRequired[Literal['auto'] | Literal['none'] | AnyModelParamsFunctionCall | None]
|
|
2253
|
+
n: NotRequired[float | None]
|
|
2254
|
+
stop: NotRequired[Sequence[str] | None]
|
|
2255
|
+
reasoning_effort: NotRequired[Literal['none', 'minimal', 'low', 'medium', 'high'] | None]
|
|
2256
|
+
verbosity: NotRequired[Literal['low', 'medium', 'high'] | None]
|
|
2257
|
+
top_k: NotRequired[float | None]
|
|
2258
|
+
stop_sequences: NotRequired[Sequence[str] | None]
|
|
2259
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
2260
|
+
reasoning_budget: NotRequired[float | None]
|
|
2261
|
+
max_tokens_to_sample: NotRequired[float | None]
|
|
1875
2262
|
"""
|
|
1876
2263
|
This is a legacy parameter that should not be used.
|
|
1877
2264
|
"""
|
|
1878
|
-
maxOutputTokens: NotRequired[
|
|
1879
|
-
topP: NotRequired[
|
|
1880
|
-
topK: NotRequired[
|
|
1881
|
-
use_cache: NotRequired[
|
|
2265
|
+
maxOutputTokens: NotRequired[float | None]
|
|
2266
|
+
topP: NotRequired[float | None]
|
|
2267
|
+
topK: NotRequired[float | None]
|
|
2268
|
+
use_cache: NotRequired[bool | None]
|
|
1882
2269
|
|
|
1883
2270
|
|
|
1884
2271
|
class AsyncScoringControlAsyncScoringControl1(TypedDict):
|
|
@@ -1886,72 +2273,72 @@ class AsyncScoringControlAsyncScoringControl1(TypedDict):
|
|
|
1886
2273
|
state: AsyncScoringState
|
|
1887
2274
|
|
|
1888
2275
|
|
|
1889
|
-
AsyncScoringControl =
|
|
1890
|
-
AsyncScoringControlAsyncScoringControl
|
|
1891
|
-
AsyncScoringControlAsyncScoringControl1
|
|
1892
|
-
AsyncScoringControlAsyncScoringControl2
|
|
1893
|
-
AsyncScoringControlAsyncScoringControl3
|
|
1894
|
-
|
|
2276
|
+
AsyncScoringControl: TypeAlias = (
|
|
2277
|
+
AsyncScoringControlAsyncScoringControl
|
|
2278
|
+
| AsyncScoringControlAsyncScoringControl1
|
|
2279
|
+
| AsyncScoringControlAsyncScoringControl2
|
|
2280
|
+
| AsyncScoringControlAsyncScoringControl3
|
|
2281
|
+
)
|
|
1895
2282
|
|
|
1896
2283
|
|
|
1897
|
-
AttachmentReference =
|
|
2284
|
+
AttachmentReference: TypeAlias = BraintrustAttachmentReference | ExternalAttachmentReference
|
|
1898
2285
|
|
|
1899
2286
|
|
|
1900
2287
|
class AttachmentStatus(TypedDict):
|
|
1901
2288
|
upload_status: UploadStatus
|
|
1902
|
-
error_message: NotRequired[
|
|
2289
|
+
error_message: NotRequired[str | None]
|
|
1903
2290
|
"""
|
|
1904
2291
|
Describes the error encountered while uploading.
|
|
1905
2292
|
"""
|
|
1906
2293
|
|
|
1907
2294
|
|
|
1908
|
-
ChatCompletionContentPart =
|
|
1909
|
-
ChatCompletionContentPartTextWithTitle
|
|
1910
|
-
ChatCompletionContentPartImageWithTitle
|
|
1911
|
-
ChatCompletionContentPartFileWithTitle
|
|
1912
|
-
|
|
2295
|
+
ChatCompletionContentPart: TypeAlias = (
|
|
2296
|
+
ChatCompletionContentPartTextWithTitle
|
|
2297
|
+
| ChatCompletionContentPartImageWithTitle
|
|
2298
|
+
| ChatCompletionContentPartFileWithTitle
|
|
2299
|
+
)
|
|
1913
2300
|
|
|
1914
2301
|
|
|
1915
2302
|
class ChatCompletionMessageParamChatCompletionMessageParam1(TypedDict):
|
|
1916
|
-
content:
|
|
2303
|
+
content: str | Sequence[ChatCompletionContentPart]
|
|
1917
2304
|
role: Literal['user']
|
|
1918
|
-
name: NotRequired[
|
|
2305
|
+
name: NotRequired[str | None]
|
|
1919
2306
|
|
|
1920
2307
|
|
|
1921
2308
|
class ChatCompletionMessageParamChatCompletionMessageParam2(TypedDict):
|
|
1922
2309
|
role: Literal['assistant']
|
|
1923
|
-
content: NotRequired[
|
|
1924
|
-
function_call: NotRequired[
|
|
1925
|
-
name: NotRequired[
|
|
1926
|
-
tool_calls: NotRequired[
|
|
1927
|
-
reasoning: NotRequired[
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
ChatCompletionMessageParam =
|
|
1931
|
-
ChatCompletionMessageParamChatCompletionMessageParam
|
|
1932
|
-
ChatCompletionMessageParamChatCompletionMessageParam1
|
|
1933
|
-
ChatCompletionMessageParamChatCompletionMessageParam2
|
|
1934
|
-
ChatCompletionMessageParamChatCompletionMessageParam3
|
|
1935
|
-
ChatCompletionMessageParamChatCompletionMessageParam4
|
|
1936
|
-
ChatCompletionMessageParamChatCompletionMessageParam5
|
|
1937
|
-
ChatCompletionMessageParamChatCompletionMessageParam6
|
|
1938
|
-
|
|
2310
|
+
content: NotRequired[str | Sequence[ChatCompletionContentPartText] | None]
|
|
2311
|
+
function_call: NotRequired[ChatCompletionMessageParamChatCompletionMessageParam2FunctionCall | None]
|
|
2312
|
+
name: NotRequired[str | None]
|
|
2313
|
+
tool_calls: NotRequired[Sequence[ChatCompletionMessageToolCall] | None]
|
|
2314
|
+
reasoning: NotRequired[Sequence[ChatCompletionMessageReasoning] | None]
|
|
2315
|
+
|
|
2316
|
+
|
|
2317
|
+
ChatCompletionMessageParam: TypeAlias = (
|
|
2318
|
+
ChatCompletionMessageParamChatCompletionMessageParam
|
|
2319
|
+
| ChatCompletionMessageParamChatCompletionMessageParam1
|
|
2320
|
+
| ChatCompletionMessageParamChatCompletionMessageParam2
|
|
2321
|
+
| ChatCompletionMessageParamChatCompletionMessageParam3
|
|
2322
|
+
| ChatCompletionMessageParamChatCompletionMessageParam4
|
|
2323
|
+
| ChatCompletionMessageParamChatCompletionMessageParam5
|
|
2324
|
+
| ChatCompletionMessageParamChatCompletionMessageParam6
|
|
2325
|
+
)
|
|
1939
2326
|
|
|
1940
2327
|
|
|
1941
2328
|
class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1(TypedDict):
|
|
1942
|
-
content:
|
|
2329
|
+
content: str | Sequence[ChatCompletionContentPart]
|
|
1943
2330
|
role: Literal['user']
|
|
1944
|
-
name: NotRequired[
|
|
2331
|
+
name: NotRequired[str | None]
|
|
1945
2332
|
|
|
1946
2333
|
|
|
1947
|
-
ChatCompletionOpenAIMessageParam =
|
|
1948
|
-
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam
|
|
1949
|
-
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1
|
|
1950
|
-
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2
|
|
1951
|
-
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam3
|
|
1952
|
-
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam4
|
|
1953
|
-
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam5
|
|
1954
|
-
|
|
2334
|
+
ChatCompletionOpenAIMessageParam: TypeAlias = (
|
|
2335
|
+
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam
|
|
2336
|
+
| ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1
|
|
2337
|
+
| ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2
|
|
2338
|
+
| ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam3
|
|
2339
|
+
| ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam4
|
|
2340
|
+
| ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam5
|
|
2341
|
+
)
|
|
1955
2342
|
|
|
1956
2343
|
|
|
1957
2344
|
class DatasetEvent(TypedDict):
|
|
@@ -1967,7 +2354,7 @@ class DatasetEvent(TypedDict):
|
|
|
1967
2354
|
"""
|
|
1968
2355
|
The timestamp the dataset event was created
|
|
1969
2356
|
"""
|
|
1970
|
-
_pagination_key: NotRequired[
|
|
2357
|
+
_pagination_key: NotRequired[str | None]
|
|
1971
2358
|
"""
|
|
1972
2359
|
A stable, time-ordered key that can be used to paginate over dataset events. This field is auto-generated by Braintrust and only exists in Brainstore.
|
|
1973
2360
|
"""
|
|
@@ -1979,19 +2366,19 @@ class DatasetEvent(TypedDict):
|
|
|
1979
2366
|
"""
|
|
1980
2367
|
Unique identifier for the dataset
|
|
1981
2368
|
"""
|
|
1982
|
-
input: NotRequired[
|
|
2369
|
+
input: NotRequired[Any | None]
|
|
1983
2370
|
"""
|
|
1984
2371
|
The argument that uniquely define an input case (an arbitrary, JSON serializable object)
|
|
1985
2372
|
"""
|
|
1986
|
-
expected: NotRequired[
|
|
2373
|
+
expected: NotRequired[Any | None]
|
|
1987
2374
|
"""
|
|
1988
2375
|
The output of your application, including post-processing (an arbitrary, JSON serializable object)
|
|
1989
2376
|
"""
|
|
1990
|
-
metadata: NotRequired[
|
|
2377
|
+
metadata: NotRequired[DatasetEventMetadata | None]
|
|
1991
2378
|
"""
|
|
1992
2379
|
A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings
|
|
1993
2380
|
"""
|
|
1994
|
-
tags: NotRequired[
|
|
2381
|
+
tags: NotRequired[Sequence[str] | None]
|
|
1995
2382
|
"""
|
|
1996
2383
|
A list of tags to log
|
|
1997
2384
|
"""
|
|
@@ -2003,16 +2390,16 @@ class DatasetEvent(TypedDict):
|
|
|
2003
2390
|
"""
|
|
2004
2391
|
A unique identifier for the trace this dataset event belongs to
|
|
2005
2392
|
"""
|
|
2006
|
-
is_root: NotRequired[
|
|
2393
|
+
is_root: NotRequired[bool | None]
|
|
2007
2394
|
"""
|
|
2008
2395
|
Whether this span is a root span
|
|
2009
2396
|
"""
|
|
2010
|
-
origin: NotRequired[
|
|
2011
|
-
comments: NotRequired[
|
|
2397
|
+
origin: NotRequired[ObjectReferenceNullish | None]
|
|
2398
|
+
comments: NotRequired[Sequence[Any] | None]
|
|
2012
2399
|
"""
|
|
2013
2400
|
Optional list of comments attached to this event
|
|
2014
2401
|
"""
|
|
2015
|
-
audit_data: NotRequired[
|
|
2402
|
+
audit_data: NotRequired[Sequence[Any] | None]
|
|
2016
2403
|
"""
|
|
2017
2404
|
Optional list of audit entries attached to this event
|
|
2018
2405
|
"""
|
|
@@ -2031,32 +2418,32 @@ class Experiment(TypedDict):
|
|
|
2031
2418
|
"""
|
|
2032
2419
|
Name of the experiment. Within a project, experiment names are unique
|
|
2033
2420
|
"""
|
|
2034
|
-
description: NotRequired[
|
|
2421
|
+
description: NotRequired[str | None]
|
|
2035
2422
|
"""
|
|
2036
2423
|
Textual description of the experiment
|
|
2037
2424
|
"""
|
|
2038
|
-
created: NotRequired[
|
|
2425
|
+
created: NotRequired[str | None]
|
|
2039
2426
|
"""
|
|
2040
2427
|
Date of experiment creation
|
|
2041
2428
|
"""
|
|
2042
|
-
repo_info: NotRequired[
|
|
2043
|
-
commit: NotRequired[
|
|
2429
|
+
repo_info: NotRequired[RepoInfo | None]
|
|
2430
|
+
commit: NotRequired[str | None]
|
|
2044
2431
|
"""
|
|
2045
2432
|
Commit, taken directly from `repo_info.commit`
|
|
2046
2433
|
"""
|
|
2047
|
-
base_exp_id: NotRequired[
|
|
2434
|
+
base_exp_id: NotRequired[str | None]
|
|
2048
2435
|
"""
|
|
2049
2436
|
Id of default base experiment to compare against when viewing this experiment
|
|
2050
2437
|
"""
|
|
2051
|
-
deleted_at: NotRequired[
|
|
2438
|
+
deleted_at: NotRequired[str | None]
|
|
2052
2439
|
"""
|
|
2053
2440
|
Date of experiment deletion, or null if the experiment is still active
|
|
2054
2441
|
"""
|
|
2055
|
-
dataset_id: NotRequired[
|
|
2442
|
+
dataset_id: NotRequired[str | None]
|
|
2056
2443
|
"""
|
|
2057
2444
|
Identifier of the linked dataset, or null if the experiment is not linked to a dataset
|
|
2058
2445
|
"""
|
|
2059
|
-
dataset_version: NotRequired[
|
|
2446
|
+
dataset_version: NotRequired[str | None]
|
|
2060
2447
|
"""
|
|
2061
2448
|
Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified.
|
|
2062
2449
|
"""
|
|
@@ -2064,51 +2451,126 @@ class Experiment(TypedDict):
|
|
|
2064
2451
|
"""
|
|
2065
2452
|
Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization
|
|
2066
2453
|
"""
|
|
2067
|
-
user_id: NotRequired[
|
|
2454
|
+
user_id: NotRequired[str | None]
|
|
2068
2455
|
"""
|
|
2069
2456
|
Identifies the user who created the experiment
|
|
2070
2457
|
"""
|
|
2071
|
-
metadata: NotRequired[
|
|
2458
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
2072
2459
|
"""
|
|
2073
2460
|
User-controlled metadata about the experiment
|
|
2074
2461
|
"""
|
|
2075
|
-
tags: NotRequired[
|
|
2462
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2076
2463
|
"""
|
|
2077
2464
|
A list of tags for the experiment
|
|
2078
2465
|
"""
|
|
2079
2466
|
|
|
2080
2467
|
|
|
2468
|
+
class ExtendedSavedFunctionIdExtendedSavedFunctionId1(TypedDict):
|
|
2469
|
+
type: Literal['global']
|
|
2470
|
+
name: str
|
|
2471
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
2472
|
+
|
|
2473
|
+
|
|
2474
|
+
ExtendedSavedFunctionId: TypeAlias = (
|
|
2475
|
+
ExtendedSavedFunctionIdExtendedSavedFunctionId
|
|
2476
|
+
| ExtendedSavedFunctionIdExtendedSavedFunctionId1
|
|
2477
|
+
| ExtendedSavedFunctionIdExtendedSavedFunctionId2
|
|
2478
|
+
)
|
|
2479
|
+
|
|
2480
|
+
|
|
2481
|
+
class PreprocessorPreprocessor1(TypedDict):
|
|
2482
|
+
type: Literal['global']
|
|
2483
|
+
name: str
|
|
2484
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
2485
|
+
|
|
2486
|
+
|
|
2487
|
+
class PreprocessorPreprocessor4(PreprocessorPreprocessor1, PreprocessorPreprocessor2):
|
|
2488
|
+
pass
|
|
2489
|
+
|
|
2490
|
+
|
|
2491
|
+
Preprocessor: TypeAlias = PreprocessorPreprocessor3 | PreprocessorPreprocessor4
|
|
2492
|
+
|
|
2493
|
+
|
|
2494
|
+
class FacetData(TypedDict):
|
|
2495
|
+
type: Literal['facet']
|
|
2496
|
+
preprocessor: NotRequired[Preprocessor | None]
|
|
2497
|
+
prompt: str
|
|
2498
|
+
"""
|
|
2499
|
+
The prompt to use for LLM extraction. The preprocessed text will be provided as context.
|
|
2500
|
+
"""
|
|
2501
|
+
model: NotRequired[str | None]
|
|
2502
|
+
"""
|
|
2503
|
+
The model to use for facet extraction
|
|
2504
|
+
"""
|
|
2505
|
+
no_match_pattern: NotRequired[str | None]
|
|
2506
|
+
"""
|
|
2507
|
+
Regex pattern to identify outputs that do not match the facet. If the output matches, the facet will be saved as 'no_match'
|
|
2508
|
+
"""
|
|
2509
|
+
|
|
2510
|
+
|
|
2511
|
+
class FunctionDataFunctionData3(TypedDict):
|
|
2512
|
+
type: Literal['global']
|
|
2513
|
+
name: str
|
|
2514
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
2515
|
+
config: NotRequired[Mapping[str, Any] | None]
|
|
2516
|
+
"""
|
|
2517
|
+
Configuration options to pass to the global function (e.g., for preprocessor customization)
|
|
2518
|
+
"""
|
|
2519
|
+
|
|
2520
|
+
|
|
2521
|
+
class FunctionIdFunctionId2(TypedDict):
|
|
2522
|
+
global_function: str
|
|
2523
|
+
"""
|
|
2524
|
+
The name of the global function. Currently, the global namespace includes the functions in autoevals
|
|
2525
|
+
"""
|
|
2526
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
2527
|
+
|
|
2528
|
+
|
|
2529
|
+
class InvokeFunctionInvokeFunction2(TypedDict):
|
|
2530
|
+
global_function: str
|
|
2531
|
+
"""
|
|
2532
|
+
The name of the global function. Currently, the global namespace includes the functions in autoevals
|
|
2533
|
+
"""
|
|
2534
|
+
function_type: NotRequired[NullableFunctionTypeEnum | None]
|
|
2535
|
+
|
|
2536
|
+
|
|
2537
|
+
InvokeScope: TypeAlias = SpanScope | TraceScope
|
|
2538
|
+
"""
|
|
2539
|
+
The scope at which to operate (span or trace)
|
|
2540
|
+
"""
|
|
2541
|
+
|
|
2542
|
+
|
|
2081
2543
|
class ModelParamsModelParams(TypedDict):
|
|
2082
|
-
use_cache: NotRequired[
|
|
2083
|
-
reasoning_enabled: NotRequired[
|
|
2084
|
-
reasoning_budget: NotRequired[
|
|
2085
|
-
temperature: NotRequired[
|
|
2086
|
-
top_p: NotRequired[
|
|
2087
|
-
max_tokens: NotRequired[
|
|
2088
|
-
max_completion_tokens: NotRequired[
|
|
2544
|
+
use_cache: NotRequired[bool | None]
|
|
2545
|
+
reasoning_enabled: NotRequired[bool | None]
|
|
2546
|
+
reasoning_budget: NotRequired[float | None]
|
|
2547
|
+
temperature: NotRequired[float | None]
|
|
2548
|
+
top_p: NotRequired[float | None]
|
|
2549
|
+
max_tokens: NotRequired[float | None]
|
|
2550
|
+
max_completion_tokens: NotRequired[float | None]
|
|
2089
2551
|
"""
|
|
2090
2552
|
The successor to max_tokens
|
|
2091
2553
|
"""
|
|
2092
|
-
frequency_penalty: NotRequired[
|
|
2093
|
-
presence_penalty: NotRequired[
|
|
2094
|
-
response_format: NotRequired[
|
|
2554
|
+
frequency_penalty: NotRequired[float | None]
|
|
2555
|
+
presence_penalty: NotRequired[float | None]
|
|
2556
|
+
response_format: NotRequired[ResponseFormatNullish | None]
|
|
2095
2557
|
tool_choice: NotRequired[
|
|
2096
|
-
|
|
2558
|
+
Literal['auto'] | Literal['none'] | Literal['required'] | ModelParamsModelParamsToolChoice
|
|
2097
2559
|
]
|
|
2098
|
-
function_call: NotRequired[
|
|
2099
|
-
n: NotRequired[
|
|
2100
|
-
stop: NotRequired[
|
|
2101
|
-
reasoning_effort: NotRequired[
|
|
2102
|
-
verbosity: NotRequired[
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
ModelParams =
|
|
2106
|
-
ModelParamsModelParams
|
|
2107
|
-
ModelParamsModelParams1
|
|
2108
|
-
ModelParamsModelParams2
|
|
2109
|
-
ModelParamsModelParams3
|
|
2110
|
-
ModelParamsModelParams4
|
|
2111
|
-
|
|
2560
|
+
function_call: NotRequired[Literal['auto'] | Literal['none'] | ModelParamsModelParamsFunctionCall | None]
|
|
2561
|
+
n: NotRequired[float | None]
|
|
2562
|
+
stop: NotRequired[Sequence[str] | None]
|
|
2563
|
+
reasoning_effort: NotRequired[Literal['none', 'minimal', 'low', 'medium', 'high'] | None]
|
|
2564
|
+
verbosity: NotRequired[Literal['low', 'medium', 'high'] | None]
|
|
2565
|
+
|
|
2566
|
+
|
|
2567
|
+
ModelParams: TypeAlias = (
|
|
2568
|
+
ModelParamsModelParams
|
|
2569
|
+
| ModelParamsModelParams1
|
|
2570
|
+
| ModelParamsModelParams2
|
|
2571
|
+
| ModelParamsModelParams3
|
|
2572
|
+
| ModelParamsModelParams4
|
|
2573
|
+
)
|
|
2112
2574
|
|
|
2113
2575
|
|
|
2114
2576
|
class OnlineScoreConfig(TypedDict):
|
|
@@ -2120,19 +2582,19 @@ class OnlineScoreConfig(TypedDict):
|
|
|
2120
2582
|
"""
|
|
2121
2583
|
The list of scorers to use for online scoring
|
|
2122
2584
|
"""
|
|
2123
|
-
btql_filter: NotRequired[
|
|
2585
|
+
btql_filter: NotRequired[str | None]
|
|
2124
2586
|
"""
|
|
2125
2587
|
Filter logs using BTQL
|
|
2126
2588
|
"""
|
|
2127
|
-
apply_to_root_span: NotRequired[
|
|
2589
|
+
apply_to_root_span: NotRequired[bool | None]
|
|
2128
2590
|
"""
|
|
2129
2591
|
Whether to trigger online scoring on the root span of each trace
|
|
2130
2592
|
"""
|
|
2131
|
-
apply_to_span_names: NotRequired[
|
|
2593
|
+
apply_to_span_names: NotRequired[Sequence[str] | None]
|
|
2132
2594
|
"""
|
|
2133
2595
|
Trigger online scoring on any spans with a name in this list
|
|
2134
2596
|
"""
|
|
2135
|
-
skip_logging: NotRequired[
|
|
2597
|
+
skip_logging: NotRequired[bool | None]
|
|
2136
2598
|
"""
|
|
2137
2599
|
Whether to skip adding scorer spans when computing scores
|
|
2138
2600
|
"""
|
|
@@ -2151,19 +2613,23 @@ class Project(TypedDict):
|
|
|
2151
2613
|
"""
|
|
2152
2614
|
Name of the project
|
|
2153
2615
|
"""
|
|
2154
|
-
|
|
2616
|
+
description: NotRequired[str | None]
|
|
2617
|
+
"""
|
|
2618
|
+
Textual description of the project
|
|
2619
|
+
"""
|
|
2620
|
+
created: NotRequired[str | None]
|
|
2155
2621
|
"""
|
|
2156
2622
|
Date of project creation
|
|
2157
2623
|
"""
|
|
2158
|
-
deleted_at: NotRequired[
|
|
2624
|
+
deleted_at: NotRequired[str | None]
|
|
2159
2625
|
"""
|
|
2160
2626
|
Date of project deletion, or null if the project is still active
|
|
2161
2627
|
"""
|
|
2162
|
-
user_id: NotRequired[
|
|
2628
|
+
user_id: NotRequired[str | None]
|
|
2163
2629
|
"""
|
|
2164
2630
|
Identifies the user who created the project
|
|
2165
2631
|
"""
|
|
2166
|
-
settings: NotRequired[
|
|
2632
|
+
settings: NotRequired[ProjectSettings | None]
|
|
2167
2633
|
|
|
2168
2634
|
|
|
2169
2635
|
class ProjectAutomationConfig2(TypedDict):
|
|
@@ -2187,11 +2653,11 @@ class ProjectAutomation(TypedDict):
|
|
|
2187
2653
|
"""
|
|
2188
2654
|
Unique identifier for the project that the project automation belongs under
|
|
2189
2655
|
"""
|
|
2190
|
-
user_id: NotRequired[
|
|
2656
|
+
user_id: NotRequired[str | None]
|
|
2191
2657
|
"""
|
|
2192
2658
|
Identifies the user who created the project automation
|
|
2193
2659
|
"""
|
|
2194
|
-
created: NotRequired[
|
|
2660
|
+
created: NotRequired[str | None]
|
|
2195
2661
|
"""
|
|
2196
2662
|
Date of project automation creation
|
|
2197
2663
|
"""
|
|
@@ -2199,55 +2665,55 @@ class ProjectAutomation(TypedDict):
|
|
|
2199
2665
|
"""
|
|
2200
2666
|
Name of the project automation
|
|
2201
2667
|
"""
|
|
2202
|
-
description: NotRequired[
|
|
2668
|
+
description: NotRequired[str | None]
|
|
2203
2669
|
"""
|
|
2204
2670
|
Textual description of the project automation
|
|
2205
2671
|
"""
|
|
2206
|
-
config:
|
|
2672
|
+
config: ProjectAutomationConfig | ProjectAutomationConfig1 | ProjectAutomationConfig2 | ProjectAutomationConfig3
|
|
2207
2673
|
"""
|
|
2208
2674
|
The configuration for the automation rule
|
|
2209
2675
|
"""
|
|
2210
2676
|
|
|
2211
2677
|
|
|
2212
|
-
ProjectScoreCategories =
|
|
2678
|
+
ProjectScoreCategories: TypeAlias = Sequence[ProjectScoreCategory] | Mapping[str, float] | Sequence[str] | None
|
|
2213
2679
|
|
|
2214
2680
|
|
|
2215
2681
|
class ProjectScoreConfig(TypedDict):
|
|
2216
|
-
multi_select: NotRequired[
|
|
2217
|
-
destination: NotRequired[
|
|
2218
|
-
online: NotRequired[
|
|
2682
|
+
multi_select: NotRequired[bool | None]
|
|
2683
|
+
destination: NotRequired[str | None]
|
|
2684
|
+
online: NotRequired[OnlineScoreConfig | None]
|
|
2219
2685
|
|
|
2220
2686
|
|
|
2221
2687
|
class PromptBlockDataPromptBlockData(TypedDict):
|
|
2222
2688
|
type: Literal['chat']
|
|
2223
2689
|
messages: Sequence[ChatCompletionMessageParam]
|
|
2224
|
-
tools: NotRequired[
|
|
2690
|
+
tools: NotRequired[str | None]
|
|
2225
2691
|
|
|
2226
2692
|
|
|
2227
|
-
PromptBlockData =
|
|
2693
|
+
PromptBlockData: TypeAlias = PromptBlockDataPromptBlockData | PromptBlockDataPromptBlockData1
|
|
2228
2694
|
|
|
2229
2695
|
|
|
2230
2696
|
class PromptBlockDataNullishPromptBlockDataNullish(TypedDict):
|
|
2231
2697
|
type: Literal['chat']
|
|
2232
2698
|
messages: Sequence[ChatCompletionMessageParam]
|
|
2233
|
-
tools: NotRequired[
|
|
2699
|
+
tools: NotRequired[str | None]
|
|
2234
2700
|
|
|
2235
2701
|
|
|
2236
|
-
PromptBlockDataNullish =
|
|
2237
|
-
|
|
2238
|
-
|
|
2702
|
+
PromptBlockDataNullish: TypeAlias = (
|
|
2703
|
+
PromptBlockDataNullishPromptBlockDataNullish | PromptBlockDataNullishPromptBlockDataNullish1 | None
|
|
2704
|
+
)
|
|
2239
2705
|
|
|
2240
2706
|
|
|
2241
2707
|
class PromptOptions(TypedDict):
|
|
2242
|
-
model: NotRequired[
|
|
2243
|
-
params: NotRequired[
|
|
2244
|
-
position: NotRequired[
|
|
2708
|
+
model: NotRequired[str | None]
|
|
2709
|
+
params: NotRequired[ModelParams | None]
|
|
2710
|
+
position: NotRequired[str | None]
|
|
2245
2711
|
|
|
2246
2712
|
|
|
2247
2713
|
class PromptOptionsNullish(TypedDict):
|
|
2248
|
-
model: NotRequired[
|
|
2249
|
-
params: NotRequired[
|
|
2250
|
-
position: NotRequired[
|
|
2714
|
+
model: NotRequired[str | None]
|
|
2715
|
+
params: NotRequired[ModelParams | None]
|
|
2716
|
+
position: NotRequired[str | None]
|
|
2251
2717
|
|
|
2252
2718
|
|
|
2253
2719
|
class ResponseFormatResponseFormat1(TypedDict):
|
|
@@ -2255,20 +2721,22 @@ class ResponseFormatResponseFormat1(TypedDict):
|
|
|
2255
2721
|
json_schema: ResponseFormatJsonSchema
|
|
2256
2722
|
|
|
2257
2723
|
|
|
2258
|
-
ResponseFormat =
|
|
2724
|
+
ResponseFormat: TypeAlias = (
|
|
2725
|
+
ResponseFormatResponseFormat | ResponseFormatResponseFormat1 | ResponseFormatResponseFormat2
|
|
2726
|
+
)
|
|
2259
2727
|
|
|
2260
2728
|
|
|
2261
2729
|
class SpanAttributes(TypedDict):
|
|
2262
|
-
name: NotRequired[
|
|
2730
|
+
name: NotRequired[str | None]
|
|
2263
2731
|
"""
|
|
2264
2732
|
Name of the span, for display purposes only
|
|
2265
2733
|
"""
|
|
2266
|
-
type: NotRequired[
|
|
2734
|
+
type: NotRequired[SpanType | None]
|
|
2267
2735
|
|
|
2268
2736
|
|
|
2269
2737
|
class ViewData(TypedDict):
|
|
2270
|
-
search: NotRequired[
|
|
2271
|
-
custom_charts: NotRequired[
|
|
2738
|
+
search: NotRequired[ViewDataSearch | None]
|
|
2739
|
+
custom_charts: NotRequired[Any | None]
|
|
2272
2740
|
|
|
2273
2741
|
|
|
2274
2742
|
class ExperimentEvent(TypedDict):
|
|
@@ -2284,7 +2752,7 @@ class ExperimentEvent(TypedDict):
|
|
|
2284
2752
|
"""
|
|
2285
2753
|
The timestamp the experiment event was created
|
|
2286
2754
|
"""
|
|
2287
|
-
_pagination_key: NotRequired[
|
|
2755
|
+
_pagination_key: NotRequired[str | None]
|
|
2288
2756
|
"""
|
|
2289
2757
|
A stable, time-ordered key that can be used to paginate over experiment events. This field is auto-generated by Braintrust and only exists in Brainstore.
|
|
2290
2758
|
"""
|
|
@@ -2296,39 +2764,39 @@ class ExperimentEvent(TypedDict):
|
|
|
2296
2764
|
"""
|
|
2297
2765
|
Unique identifier for the experiment
|
|
2298
2766
|
"""
|
|
2299
|
-
input: NotRequired[
|
|
2767
|
+
input: NotRequired[Any | None]
|
|
2300
2768
|
"""
|
|
2301
2769
|
The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical
|
|
2302
2770
|
"""
|
|
2303
|
-
output: NotRequired[
|
|
2771
|
+
output: NotRequired[Any | None]
|
|
2304
2772
|
"""
|
|
2305
2773
|
The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question
|
|
2306
2774
|
"""
|
|
2307
|
-
expected: NotRequired[
|
|
2775
|
+
expected: NotRequired[Any | None]
|
|
2308
2776
|
"""
|
|
2309
2777
|
The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models
|
|
2310
2778
|
"""
|
|
2311
|
-
error: NotRequired[
|
|
2779
|
+
error: NotRequired[Any | None]
|
|
2312
2780
|
"""
|
|
2313
2781
|
The error that occurred, if any.
|
|
2314
2782
|
"""
|
|
2315
|
-
scores: NotRequired[
|
|
2783
|
+
scores: NotRequired[Mapping[str, Any] | None]
|
|
2316
2784
|
"""
|
|
2317
2785
|
A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments
|
|
2318
2786
|
"""
|
|
2319
|
-
metadata: NotRequired[
|
|
2787
|
+
metadata: NotRequired[ExperimentEventMetadata | None]
|
|
2320
2788
|
"""
|
|
2321
2789
|
A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings
|
|
2322
2790
|
"""
|
|
2323
|
-
tags: NotRequired[
|
|
2791
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2324
2792
|
"""
|
|
2325
2793
|
A list of tags to log
|
|
2326
2794
|
"""
|
|
2327
|
-
metrics: NotRequired[
|
|
2795
|
+
metrics: NotRequired[ExperimentEventMetrics | None]
|
|
2328
2796
|
"""
|
|
2329
2797
|
Metrics are numerical measurements tracking the execution of the code that produced the experiment event. Use "start" and "end" to track the time span over which the experiment event was produced
|
|
2330
2798
|
"""
|
|
2331
|
-
context: NotRequired[
|
|
2799
|
+
context: NotRequired[ExperimentEventContext | None]
|
|
2332
2800
|
"""
|
|
2333
2801
|
Context is additional information about the code that produced the experiment event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the experiment event
|
|
2334
2802
|
"""
|
|
@@ -2336,7 +2804,7 @@ class ExperimentEvent(TypedDict):
|
|
|
2336
2804
|
"""
|
|
2337
2805
|
A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing
|
|
2338
2806
|
"""
|
|
2339
|
-
span_parents: NotRequired[
|
|
2807
|
+
span_parents: NotRequired[Sequence[str] | None]
|
|
2340
2808
|
"""
|
|
2341
2809
|
An array of the parent `span_ids` of this experiment event. This should be empty for the root span of a trace, and should most often contain just one parent element for subspans
|
|
2342
2810
|
"""
|
|
@@ -2344,28 +2812,28 @@ class ExperimentEvent(TypedDict):
|
|
|
2344
2812
|
"""
|
|
2345
2813
|
A unique identifier for the trace this experiment event belongs to
|
|
2346
2814
|
"""
|
|
2347
|
-
span_attributes: NotRequired[
|
|
2348
|
-
is_root: NotRequired[
|
|
2815
|
+
span_attributes: NotRequired[SpanAttributes | None]
|
|
2816
|
+
is_root: NotRequired[bool | None]
|
|
2349
2817
|
"""
|
|
2350
2818
|
Whether this span is a root span
|
|
2351
2819
|
"""
|
|
2352
|
-
origin: NotRequired[
|
|
2353
|
-
comments: NotRequired[
|
|
2820
|
+
origin: NotRequired[ObjectReferenceNullish | None]
|
|
2821
|
+
comments: NotRequired[Sequence[Any] | None]
|
|
2354
2822
|
"""
|
|
2355
2823
|
Optional list of comments attached to this event
|
|
2356
2824
|
"""
|
|
2357
|
-
audit_data: NotRequired[
|
|
2825
|
+
audit_data: NotRequired[Sequence[Any] | None]
|
|
2358
2826
|
"""
|
|
2359
2827
|
Optional list of audit entries attached to this event
|
|
2360
2828
|
"""
|
|
2361
2829
|
|
|
2362
2830
|
|
|
2363
2831
|
class GraphNodeGraphNode7(TypedDict):
|
|
2364
|
-
description: NotRequired[
|
|
2832
|
+
description: NotRequired[str | None]
|
|
2365
2833
|
"""
|
|
2366
2834
|
The description of the node
|
|
2367
2835
|
"""
|
|
2368
|
-
position: NotRequired[
|
|
2836
|
+
position: NotRequired[GraphNodeGraphNode7Position | None]
|
|
2369
2837
|
"""
|
|
2370
2838
|
The position of the node
|
|
2371
2839
|
"""
|
|
@@ -2373,16 +2841,90 @@ class GraphNodeGraphNode7(TypedDict):
|
|
|
2373
2841
|
prompt: PromptBlockData
|
|
2374
2842
|
|
|
2375
2843
|
|
|
2376
|
-
GraphNode =
|
|
2377
|
-
GraphNodeGraphNode
|
|
2378
|
-
GraphNodeGraphNode1
|
|
2379
|
-
GraphNodeGraphNode2
|
|
2380
|
-
GraphNodeGraphNode3
|
|
2381
|
-
GraphNodeGraphNode4
|
|
2382
|
-
GraphNodeGraphNode5
|
|
2383
|
-
GraphNodeGraphNode6
|
|
2384
|
-
GraphNodeGraphNode7
|
|
2385
|
-
|
|
2844
|
+
GraphNode: TypeAlias = (
|
|
2845
|
+
GraphNodeGraphNode
|
|
2846
|
+
| GraphNodeGraphNode1
|
|
2847
|
+
| GraphNodeGraphNode2
|
|
2848
|
+
| GraphNodeGraphNode3
|
|
2849
|
+
| GraphNodeGraphNode4
|
|
2850
|
+
| GraphNodeGraphNode5
|
|
2851
|
+
| GraphNodeGraphNode6
|
|
2852
|
+
| GraphNodeGraphNode7
|
|
2853
|
+
)
|
|
2854
|
+
|
|
2855
|
+
|
|
2856
|
+
class InvokeContext(TypedDict):
|
|
2857
|
+
object_type: Literal['project_logs', 'experiment', 'dataset', 'playground_logs']
|
|
2858
|
+
"""
|
|
2859
|
+
The type of object containing the span data
|
|
2860
|
+
"""
|
|
2861
|
+
object_id: str
|
|
2862
|
+
"""
|
|
2863
|
+
The ID of the object containing the span data
|
|
2864
|
+
"""
|
|
2865
|
+
scope: InvokeScope
|
|
2866
|
+
|
|
2867
|
+
|
|
2868
|
+
class InvokeFunctionInvokeFunction7(TypedDict):
|
|
2869
|
+
input: NotRequired[Any | None]
|
|
2870
|
+
"""
|
|
2871
|
+
Argument to the function, which can be any JSON serializable value
|
|
2872
|
+
"""
|
|
2873
|
+
expected: NotRequired[Any | None]
|
|
2874
|
+
"""
|
|
2875
|
+
The expected output of the function
|
|
2876
|
+
"""
|
|
2877
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
2878
|
+
"""
|
|
2879
|
+
Any relevant metadata. This will be logged and available as the `metadata` argument.
|
|
2880
|
+
"""
|
|
2881
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2882
|
+
"""
|
|
2883
|
+
Any relevant tags to log on the span.
|
|
2884
|
+
"""
|
|
2885
|
+
messages: NotRequired[Sequence[ChatCompletionMessageParam] | None]
|
|
2886
|
+
"""
|
|
2887
|
+
If the function is an LLM, additional messages to pass along to it
|
|
2888
|
+
"""
|
|
2889
|
+
context: NotRequired[InvokeContext | None]
|
|
2890
|
+
parent: NotRequired[InvokeParent | None]
|
|
2891
|
+
stream: NotRequired[bool | None]
|
|
2892
|
+
"""
|
|
2893
|
+
Whether to stream the response. If true, results will be returned in the Braintrust SSE format.
|
|
2894
|
+
"""
|
|
2895
|
+
mode: NotRequired[StreamingMode | None]
|
|
2896
|
+
strict: NotRequired[bool | None]
|
|
2897
|
+
"""
|
|
2898
|
+
If true, throw an error if one of the variables in the prompt is not present in the input
|
|
2899
|
+
"""
|
|
2900
|
+
mcp_auth: NotRequired[Mapping[str, InvokeFunctionMcpAuth] | None]
|
|
2901
|
+
"""
|
|
2902
|
+
Map of MCP server URL to auth credentials
|
|
2903
|
+
"""
|
|
2904
|
+
overrides: NotRequired[Mapping[str, Any] | None]
|
|
2905
|
+
"""
|
|
2906
|
+
Partial function definition to merge with the function being invoked. Fields are validated against the function type's schema at runtime. For facets: { preprocessor?, prompt?, model? }. For prompts: { model?, ... }.
|
|
2907
|
+
"""
|
|
2908
|
+
|
|
2909
|
+
|
|
2910
|
+
class InvokeFunctionInvokeFunction8(InvokeFunctionInvokeFunction, InvokeFunctionInvokeFunction7):
|
|
2911
|
+
pass
|
|
2912
|
+
|
|
2913
|
+
|
|
2914
|
+
class InvokeFunctionInvokeFunction9(InvokeFunctionInvokeFunction1, InvokeFunctionInvokeFunction7):
|
|
2915
|
+
pass
|
|
2916
|
+
|
|
2917
|
+
|
|
2918
|
+
class InvokeFunctionInvokeFunction10(InvokeFunctionInvokeFunction2, InvokeFunctionInvokeFunction7):
|
|
2919
|
+
pass
|
|
2920
|
+
|
|
2921
|
+
|
|
2922
|
+
class InvokeFunctionInvokeFunction11(InvokeFunctionInvokeFunction3, InvokeFunctionInvokeFunction7):
|
|
2923
|
+
pass
|
|
2924
|
+
|
|
2925
|
+
|
|
2926
|
+
class InvokeFunctionInvokeFunction12(InvokeFunctionInvokeFunction4, InvokeFunctionInvokeFunction7):
|
|
2927
|
+
pass
|
|
2386
2928
|
|
|
2387
2929
|
|
|
2388
2930
|
class ProjectLogsEvent(TypedDict):
|
|
@@ -2394,7 +2936,7 @@ class ProjectLogsEvent(TypedDict):
|
|
|
2394
2936
|
"""
|
|
2395
2937
|
The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)
|
|
2396
2938
|
"""
|
|
2397
|
-
_pagination_key: NotRequired[
|
|
2939
|
+
_pagination_key: NotRequired[str | None]
|
|
2398
2940
|
"""
|
|
2399
2941
|
A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.
|
|
2400
2942
|
"""
|
|
@@ -2414,39 +2956,39 @@ class ProjectLogsEvent(TypedDict):
|
|
|
2414
2956
|
"""
|
|
2415
2957
|
A literal 'g' which identifies the log as a project log
|
|
2416
2958
|
"""
|
|
2417
|
-
input: NotRequired[
|
|
2959
|
+
input: NotRequired[Any | None]
|
|
2418
2960
|
"""
|
|
2419
2961
|
The arguments that uniquely define a user input (an arbitrary, JSON serializable object).
|
|
2420
2962
|
"""
|
|
2421
|
-
output: NotRequired[
|
|
2963
|
+
output: NotRequired[Any | None]
|
|
2422
2964
|
"""
|
|
2423
2965
|
The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
2424
2966
|
"""
|
|
2425
|
-
expected: NotRequired[
|
|
2967
|
+
expected: NotRequired[Any | None]
|
|
2426
2968
|
"""
|
|
2427
2969
|
The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
2428
2970
|
"""
|
|
2429
|
-
error: NotRequired[
|
|
2971
|
+
error: NotRequired[Any | None]
|
|
2430
2972
|
"""
|
|
2431
2973
|
The error that occurred, if any.
|
|
2432
2974
|
"""
|
|
2433
|
-
scores: NotRequired[
|
|
2975
|
+
scores: NotRequired[Mapping[str, Any] | None]
|
|
2434
2976
|
"""
|
|
2435
2977
|
A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
|
|
2436
2978
|
"""
|
|
2437
|
-
metadata: NotRequired[
|
|
2979
|
+
metadata: NotRequired[ProjectLogsEventMetadata | None]
|
|
2438
2980
|
"""
|
|
2439
2981
|
A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings
|
|
2440
2982
|
"""
|
|
2441
|
-
tags: NotRequired[
|
|
2983
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2442
2984
|
"""
|
|
2443
2985
|
A list of tags to log
|
|
2444
2986
|
"""
|
|
2445
|
-
metrics: NotRequired[
|
|
2987
|
+
metrics: NotRequired[ProjectLogsEventMetrics | None]
|
|
2446
2988
|
"""
|
|
2447
2989
|
Metrics are numerical measurements tracking the execution of the code that produced the project logs event. Use "start" and "end" to track the time span over which the project logs event was produced
|
|
2448
2990
|
"""
|
|
2449
|
-
context: NotRequired[
|
|
2991
|
+
context: NotRequired[ProjectLogsEventContext | None]
|
|
2450
2992
|
"""
|
|
2451
2993
|
Context is additional information about the code that produced the project logs event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the project logs event
|
|
2452
2994
|
"""
|
|
@@ -2454,7 +2996,7 @@ class ProjectLogsEvent(TypedDict):
|
|
|
2454
2996
|
"""
|
|
2455
2997
|
A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing
|
|
2456
2998
|
"""
|
|
2457
|
-
span_parents: NotRequired[
|
|
2999
|
+
span_parents: NotRequired[Sequence[str] | None]
|
|
2458
3000
|
"""
|
|
2459
3001
|
An array of the parent `span_ids` of this project logs event. This should be empty for the root span of a trace, and should most often contain just one parent element for subspans
|
|
2460
3002
|
"""
|
|
@@ -2462,21 +3004,21 @@ class ProjectLogsEvent(TypedDict):
|
|
|
2462
3004
|
"""
|
|
2463
3005
|
A unique identifier for the trace this project logs event belongs to
|
|
2464
3006
|
"""
|
|
2465
|
-
is_root: NotRequired[
|
|
3007
|
+
is_root: NotRequired[bool | None]
|
|
2466
3008
|
"""
|
|
2467
3009
|
Whether this span is a root span
|
|
2468
3010
|
"""
|
|
2469
|
-
span_attributes: NotRequired[
|
|
2470
|
-
origin: NotRequired[
|
|
2471
|
-
comments: NotRequired[
|
|
3011
|
+
span_attributes: NotRequired[SpanAttributes | None]
|
|
3012
|
+
origin: NotRequired[ObjectReferenceNullish | None]
|
|
3013
|
+
comments: NotRequired[Sequence[Any] | None]
|
|
2472
3014
|
"""
|
|
2473
3015
|
Optional list of comments attached to this event
|
|
2474
3016
|
"""
|
|
2475
|
-
audit_data: NotRequired[
|
|
3017
|
+
audit_data: NotRequired[Sequence[Any] | None]
|
|
2476
3018
|
"""
|
|
2477
3019
|
Optional list of audit entries attached to this event
|
|
2478
3020
|
"""
|
|
2479
|
-
_async_scoring_state: NotRequired[
|
|
3021
|
+
_async_scoring_state: NotRequired[Any | None]
|
|
2480
3022
|
"""
|
|
2481
3023
|
The async scoring state for this event
|
|
2482
3024
|
"""
|
|
@@ -2492,7 +3034,7 @@ class ProjectScore(TypedDict):
|
|
|
2492
3034
|
Unique identifier for the project that the project score belongs under
|
|
2493
3035
|
"""
|
|
2494
3036
|
user_id: str
|
|
2495
|
-
created: NotRequired[
|
|
3037
|
+
created: NotRequired[str | None]
|
|
2496
3038
|
"""
|
|
2497
3039
|
Date of project score creation
|
|
2498
3040
|
"""
|
|
@@ -2500,33 +3042,67 @@ class ProjectScore(TypedDict):
|
|
|
2500
3042
|
"""
|
|
2501
3043
|
Name of the project score
|
|
2502
3044
|
"""
|
|
2503
|
-
description: NotRequired[
|
|
3045
|
+
description: NotRequired[str | None]
|
|
2504
3046
|
"""
|
|
2505
3047
|
Textual description of the project score
|
|
2506
3048
|
"""
|
|
2507
3049
|
score_type: ProjectScoreType
|
|
2508
|
-
categories: NotRequired[
|
|
2509
|
-
config: NotRequired[
|
|
2510
|
-
position: NotRequired[
|
|
3050
|
+
categories: NotRequired[ProjectScoreCategories | None]
|
|
3051
|
+
config: NotRequired[ProjectScoreConfig | None]
|
|
3052
|
+
position: NotRequired[str | None]
|
|
2511
3053
|
"""
|
|
2512
3054
|
An optional LexoRank-based string that sets the sort position for the score in the UI
|
|
2513
3055
|
"""
|
|
2514
3056
|
|
|
2515
3057
|
|
|
2516
3058
|
class PromptData(TypedDict):
|
|
2517
|
-
prompt: NotRequired[
|
|
2518
|
-
options: NotRequired[
|
|
2519
|
-
parser: NotRequired[
|
|
2520
|
-
tool_functions: NotRequired[
|
|
2521
|
-
|
|
3059
|
+
prompt: NotRequired[PromptBlockDataNullish | None]
|
|
3060
|
+
options: NotRequired[PromptOptionsNullish | None]
|
|
3061
|
+
parser: NotRequired[PromptParserNullish | None]
|
|
3062
|
+
tool_functions: NotRequired[Sequence[SavedFunctionId] | None]
|
|
3063
|
+
template_format: NotRequired[Literal['mustache', 'nunjucks', 'none'] | None]
|
|
3064
|
+
mcp: NotRequired[Mapping[str, Any] | None]
|
|
3065
|
+
origin: NotRequired[PromptDataOrigin | None]
|
|
2522
3066
|
|
|
2523
3067
|
|
|
2524
3068
|
class PromptDataNullish(TypedDict):
|
|
2525
|
-
prompt: NotRequired[
|
|
2526
|
-
options: NotRequired[
|
|
2527
|
-
parser: NotRequired[
|
|
2528
|
-
tool_functions: NotRequired[
|
|
2529
|
-
|
|
3069
|
+
prompt: NotRequired[PromptBlockDataNullish | None]
|
|
3070
|
+
options: NotRequired[PromptOptionsNullish | None]
|
|
3071
|
+
parser: NotRequired[PromptParserNullish | None]
|
|
3072
|
+
tool_functions: NotRequired[Sequence[SavedFunctionId] | None]
|
|
3073
|
+
template_format: NotRequired[Literal['mustache', 'nunjucks', 'none'] | None]
|
|
3074
|
+
mcp: NotRequired[Mapping[str, Any] | None]
|
|
3075
|
+
origin: NotRequired[PromptDataNullishOrigin | None]
|
|
3076
|
+
|
|
3077
|
+
|
|
3078
|
+
class TaskTask5(TypedDict):
|
|
3079
|
+
inline_prompt: NotRequired[PromptData | None]
|
|
3080
|
+
inline_function: Mapping[str, Any]
|
|
3081
|
+
function_type: NotRequired[FunctionTypeEnum | None]
|
|
3082
|
+
name: NotRequired[str | None]
|
|
3083
|
+
"""
|
|
3084
|
+
The name of the inline function
|
|
3085
|
+
"""
|
|
3086
|
+
|
|
3087
|
+
|
|
3088
|
+
class TaskTask6(TypedDict):
|
|
3089
|
+
inline_prompt: PromptData
|
|
3090
|
+
function_type: NotRequired[FunctionTypeEnum | None]
|
|
3091
|
+
name: NotRequired[str | None]
|
|
3092
|
+
"""
|
|
3093
|
+
The name of the inline prompt
|
|
3094
|
+
"""
|
|
3095
|
+
|
|
3096
|
+
|
|
3097
|
+
class TaskTask13(TaskTask5, TaskTask7):
|
|
3098
|
+
pass
|
|
3099
|
+
|
|
3100
|
+
|
|
3101
|
+
class TaskTask14(TaskTask6, TaskTask7):
|
|
3102
|
+
pass
|
|
3103
|
+
|
|
3104
|
+
|
|
3105
|
+
Task: TypeAlias = TaskTask8 | TaskTask9 | TaskTask10 | TaskTask11 | TaskTask12 | TaskTask13 | TaskTask14
|
|
2530
3106
|
|
|
2531
3107
|
|
|
2532
3108
|
class View(TypedDict):
|
|
@@ -2534,7 +3110,7 @@ class View(TypedDict):
|
|
|
2534
3110
|
"""
|
|
2535
3111
|
Unique identifier for the view
|
|
2536
3112
|
"""
|
|
2537
|
-
object_type:
|
|
3113
|
+
object_type: AclObjectType
|
|
2538
3114
|
object_id: str
|
|
2539
3115
|
"""
|
|
2540
3116
|
The id of the object the view applies to
|
|
@@ -2562,27 +3138,27 @@ class View(TypedDict):
|
|
|
2562
3138
|
"""
|
|
2563
3139
|
Name of the view
|
|
2564
3140
|
"""
|
|
2565
|
-
created: NotRequired[
|
|
3141
|
+
created: NotRequired[str | None]
|
|
2566
3142
|
"""
|
|
2567
3143
|
Date of view creation
|
|
2568
3144
|
"""
|
|
2569
|
-
view_data: NotRequired[
|
|
2570
|
-
options: NotRequired[
|
|
2571
|
-
user_id: NotRequired[
|
|
3145
|
+
view_data: NotRequired[ViewData | None]
|
|
3146
|
+
options: NotRequired[ViewOptions | None]
|
|
3147
|
+
user_id: NotRequired[str | None]
|
|
2572
3148
|
"""
|
|
2573
3149
|
Identifies the user who created the view
|
|
2574
3150
|
"""
|
|
2575
|
-
deleted_at: NotRequired[
|
|
3151
|
+
deleted_at: NotRequired[str | None]
|
|
2576
3152
|
"""
|
|
2577
3153
|
Date of role deletion, or null if the role is still active
|
|
2578
3154
|
"""
|
|
2579
3155
|
|
|
2580
3156
|
|
|
2581
3157
|
class FunctionIdFunctionId5(TypedDict):
|
|
2582
|
-
inline_prompt: NotRequired[
|
|
3158
|
+
inline_prompt: NotRequired[PromptData | None]
|
|
2583
3159
|
inline_function: Mapping[str, Any]
|
|
2584
|
-
function_type: NotRequired[
|
|
2585
|
-
name: NotRequired[
|
|
3160
|
+
function_type: NotRequired[FunctionTypeEnum | None]
|
|
3161
|
+
name: NotRequired[str | None]
|
|
2586
3162
|
"""
|
|
2587
3163
|
The name of the inline function
|
|
2588
3164
|
"""
|
|
@@ -2590,22 +3166,25 @@ class FunctionIdFunctionId5(TypedDict):
|
|
|
2590
3166
|
|
|
2591
3167
|
class FunctionIdFunctionId6(TypedDict):
|
|
2592
3168
|
inline_prompt: PromptData
|
|
2593
|
-
function_type: NotRequired[
|
|
2594
|
-
name: NotRequired[
|
|
3169
|
+
function_type: NotRequired[FunctionTypeEnum | None]
|
|
3170
|
+
name: NotRequired[str | None]
|
|
2595
3171
|
"""
|
|
2596
3172
|
The name of the inline prompt
|
|
2597
3173
|
"""
|
|
2598
3174
|
|
|
2599
3175
|
|
|
2600
|
-
FunctionId =
|
|
2601
|
-
FunctionIdFunctionId
|
|
2602
|
-
FunctionIdFunctionId1
|
|
2603
|
-
FunctionIdFunctionId2
|
|
2604
|
-
FunctionIdFunctionId3
|
|
2605
|
-
FunctionIdFunctionId4
|
|
2606
|
-
FunctionIdFunctionId5
|
|
2607
|
-
FunctionIdFunctionId6
|
|
2608
|
-
|
|
3176
|
+
FunctionId: TypeAlias = (
|
|
3177
|
+
FunctionIdFunctionId
|
|
3178
|
+
| FunctionIdFunctionId1
|
|
3179
|
+
| FunctionIdFunctionId2
|
|
3180
|
+
| FunctionIdFunctionId3
|
|
3181
|
+
| FunctionIdFunctionId4
|
|
3182
|
+
| FunctionIdFunctionId5
|
|
3183
|
+
| FunctionIdFunctionId6
|
|
3184
|
+
)
|
|
3185
|
+
"""
|
|
3186
|
+
Options for identifying a function
|
|
3187
|
+
"""
|
|
2609
3188
|
|
|
2610
3189
|
|
|
2611
3190
|
class GraphData(TypedDict):
|
|
@@ -2614,39 +3193,47 @@ class GraphData(TypedDict):
|
|
|
2614
3193
|
edges: Mapping[str, GraphEdge]
|
|
2615
3194
|
|
|
2616
3195
|
|
|
2617
|
-
class
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
expected: NotRequired[Optional[Any]]
|
|
2623
|
-
"""
|
|
2624
|
-
The expected output of the function
|
|
2625
|
-
"""
|
|
2626
|
-
metadata: NotRequired[Optional[Mapping[str, Any]]]
|
|
2627
|
-
"""
|
|
2628
|
-
Any relevant metadata. This will be logged and available as the `metadata` argument.
|
|
2629
|
-
"""
|
|
2630
|
-
tags: NotRequired[Optional[Sequence[str]]]
|
|
2631
|
-
"""
|
|
2632
|
-
Any relevant tags to log on the span.
|
|
2633
|
-
"""
|
|
2634
|
-
messages: NotRequired[Optional[Sequence[ChatCompletionMessageParam]]]
|
|
2635
|
-
"""
|
|
2636
|
-
If the function is an LLM, additional messages to pass along to it
|
|
2637
|
-
"""
|
|
2638
|
-
parent: NotRequired[Optional[InvokeParent]]
|
|
2639
|
-
stream: NotRequired[Optional[bool]]
|
|
3196
|
+
class InvokeFunctionInvokeFunction5(TypedDict):
|
|
3197
|
+
inline_prompt: NotRequired[PromptData | None]
|
|
3198
|
+
inline_function: Mapping[str, Any]
|
|
3199
|
+
function_type: NotRequired[FunctionTypeEnum | None]
|
|
3200
|
+
name: NotRequired[str | None]
|
|
2640
3201
|
"""
|
|
2641
|
-
|
|
3202
|
+
The name of the inline function
|
|
2642
3203
|
"""
|
|
2643
|
-
|
|
2644
|
-
|
|
3204
|
+
|
|
3205
|
+
|
|
3206
|
+
class InvokeFunctionInvokeFunction6(TypedDict):
|
|
3207
|
+
inline_prompt: PromptData
|
|
3208
|
+
function_type: NotRequired[FunctionTypeEnum | None]
|
|
3209
|
+
name: NotRequired[str | None]
|
|
2645
3210
|
"""
|
|
2646
|
-
|
|
3211
|
+
The name of the inline prompt
|
|
2647
3212
|
"""
|
|
2648
3213
|
|
|
2649
3214
|
|
|
3215
|
+
class InvokeFunctionInvokeFunction13(InvokeFunctionInvokeFunction5, InvokeFunctionInvokeFunction7):
|
|
3216
|
+
pass
|
|
3217
|
+
|
|
3218
|
+
|
|
3219
|
+
class InvokeFunctionInvokeFunction14(InvokeFunctionInvokeFunction6, InvokeFunctionInvokeFunction7):
|
|
3220
|
+
pass
|
|
3221
|
+
|
|
3222
|
+
|
|
3223
|
+
InvokeFunction: TypeAlias = (
|
|
3224
|
+
InvokeFunctionInvokeFunction8
|
|
3225
|
+
| InvokeFunctionInvokeFunction9
|
|
3226
|
+
| InvokeFunctionInvokeFunction10
|
|
3227
|
+
| InvokeFunctionInvokeFunction11
|
|
3228
|
+
| InvokeFunctionInvokeFunction12
|
|
3229
|
+
| InvokeFunctionInvokeFunction13
|
|
3230
|
+
| InvokeFunctionInvokeFunction14
|
|
3231
|
+
)
|
|
3232
|
+
"""
|
|
3233
|
+
Options for identifying a function
|
|
3234
|
+
"""
|
|
3235
|
+
|
|
3236
|
+
|
|
2650
3237
|
class Prompt(TypedDict):
|
|
2651
3238
|
id: str
|
|
2652
3239
|
"""
|
|
@@ -2676,24 +3263,24 @@ class Prompt(TypedDict):
|
|
|
2676
3263
|
"""
|
|
2677
3264
|
Unique identifier for the prompt
|
|
2678
3265
|
"""
|
|
2679
|
-
description: NotRequired[
|
|
3266
|
+
description: NotRequired[str | None]
|
|
2680
3267
|
"""
|
|
2681
3268
|
Textual description of the prompt
|
|
2682
3269
|
"""
|
|
2683
|
-
created: NotRequired[
|
|
3270
|
+
created: NotRequired[str | None]
|
|
2684
3271
|
"""
|
|
2685
3272
|
Date of prompt creation
|
|
2686
3273
|
"""
|
|
2687
|
-
prompt_data: NotRequired[
|
|
2688
|
-
tags: NotRequired[
|
|
3274
|
+
prompt_data: NotRequired[PromptDataNullish | None]
|
|
3275
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2689
3276
|
"""
|
|
2690
3277
|
A list of tags for the prompt
|
|
2691
3278
|
"""
|
|
2692
|
-
metadata: NotRequired[
|
|
3279
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
2693
3280
|
"""
|
|
2694
3281
|
User-controlled metadata about the prompt
|
|
2695
3282
|
"""
|
|
2696
|
-
function_type: NotRequired[
|
|
3283
|
+
function_type: NotRequired[FunctionTypeEnumNullish | None]
|
|
2697
3284
|
|
|
2698
3285
|
|
|
2699
3286
|
class RunEval(TypedDict):
|
|
@@ -2701,79 +3288,81 @@ class RunEval(TypedDict):
|
|
|
2701
3288
|
"""
|
|
2702
3289
|
Unique identifier for the project to run the eval in
|
|
2703
3290
|
"""
|
|
2704
|
-
data:
|
|
3291
|
+
data: RunEvalData | RunEvalData1 | RunEvalData2
|
|
2705
3292
|
"""
|
|
2706
3293
|
The dataset to use
|
|
2707
3294
|
"""
|
|
2708
|
-
task:
|
|
3295
|
+
task: Task
|
|
2709
3296
|
scores: Sequence[FunctionId]
|
|
2710
3297
|
"""
|
|
2711
3298
|
The functions to score the eval on
|
|
2712
3299
|
"""
|
|
2713
|
-
experiment_name: NotRequired[
|
|
3300
|
+
experiment_name: NotRequired[str | None]
|
|
2714
3301
|
"""
|
|
2715
3302
|
An optional name for the experiment created by this eval. If it conflicts with an existing experiment, it will be suffixed with a unique identifier.
|
|
2716
3303
|
"""
|
|
2717
|
-
metadata: NotRequired[
|
|
3304
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
2718
3305
|
"""
|
|
2719
3306
|
Optional experiment-level metadata to store about the evaluation. You can later use this to slice & dice across experiments.
|
|
2720
3307
|
"""
|
|
2721
|
-
parent: NotRequired[
|
|
2722
|
-
stream: NotRequired[
|
|
3308
|
+
parent: NotRequired[Parent | None]
|
|
3309
|
+
stream: NotRequired[bool | None]
|
|
2723
3310
|
"""
|
|
2724
3311
|
Whether to stream the results of the eval. If true, the request will return two events: one to indicate the experiment has started, and another upon completion. If false, the request will return the evaluation's summary upon completion.
|
|
2725
3312
|
"""
|
|
2726
|
-
trial_count: NotRequired[
|
|
3313
|
+
trial_count: NotRequired[float | None]
|
|
2727
3314
|
"""
|
|
2728
3315
|
The number of times to run the evaluator per input. This is useful for evaluating applications that have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results.
|
|
2729
3316
|
"""
|
|
2730
|
-
is_public: NotRequired[
|
|
3317
|
+
is_public: NotRequired[bool | None]
|
|
2731
3318
|
"""
|
|
2732
3319
|
Whether the experiment should be public. Defaults to false.
|
|
2733
3320
|
"""
|
|
2734
|
-
timeout: NotRequired[
|
|
3321
|
+
timeout: NotRequired[float | None]
|
|
2735
3322
|
"""
|
|
2736
3323
|
The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout.
|
|
2737
3324
|
"""
|
|
2738
|
-
max_concurrency: NotRequired[
|
|
3325
|
+
max_concurrency: NotRequired[float | None]
|
|
2739
3326
|
"""
|
|
2740
3327
|
The maximum number of tasks/scorers that will be run concurrently. Defaults to 10. If null is provided, no max concurrency will be used.
|
|
2741
3328
|
"""
|
|
2742
|
-
base_experiment_name: NotRequired[
|
|
3329
|
+
base_experiment_name: NotRequired[str | None]
|
|
2743
3330
|
"""
|
|
2744
3331
|
An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this experiment.
|
|
2745
3332
|
"""
|
|
2746
|
-
base_experiment_id: NotRequired[
|
|
3333
|
+
base_experiment_id: NotRequired[str | None]
|
|
2747
3334
|
"""
|
|
2748
3335
|
An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this experiment.
|
|
2749
3336
|
"""
|
|
2750
|
-
git_metadata_settings: NotRequired[
|
|
2751
|
-
repo_info: NotRequired[
|
|
2752
|
-
strict: NotRequired[
|
|
3337
|
+
git_metadata_settings: NotRequired[GitMetadataSettings | None]
|
|
3338
|
+
repo_info: NotRequired[RepoInfo | None]
|
|
3339
|
+
strict: NotRequired[bool | None]
|
|
2753
3340
|
"""
|
|
2754
3341
|
If true, throw an error if one of the variables in the prompt is not present in the input
|
|
2755
3342
|
"""
|
|
2756
|
-
stop_token: NotRequired[
|
|
3343
|
+
stop_token: NotRequired[str | None]
|
|
2757
3344
|
"""
|
|
2758
3345
|
The token to stop the run
|
|
2759
3346
|
"""
|
|
2760
|
-
extra_messages: NotRequired[
|
|
3347
|
+
extra_messages: NotRequired[str | None]
|
|
2761
3348
|
"""
|
|
2762
3349
|
A template path of extra messages to append to the conversion. These messages will be appended to the end of the conversation, after the last message.
|
|
2763
3350
|
"""
|
|
2764
|
-
tags: NotRequired[
|
|
3351
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2765
3352
|
"""
|
|
2766
3353
|
Optional tags that will be added to the experiment.
|
|
2767
3354
|
"""
|
|
3355
|
+
mcp_auth: NotRequired[Mapping[str, RunEvalMcpAuth] | None]
|
|
2768
3356
|
|
|
2769
3357
|
|
|
2770
|
-
FunctionData =
|
|
2771
|
-
FunctionDataFunctionData
|
|
2772
|
-
FunctionDataFunctionData1
|
|
2773
|
-
GraphData
|
|
2774
|
-
FunctionDataFunctionData2
|
|
2775
|
-
FunctionDataFunctionData3
|
|
2776
|
-
|
|
3358
|
+
FunctionData: TypeAlias = (
|
|
3359
|
+
FunctionDataFunctionData
|
|
3360
|
+
| FunctionDataFunctionData1
|
|
3361
|
+
| GraphData
|
|
3362
|
+
| FunctionDataFunctionData2
|
|
3363
|
+
| FunctionDataFunctionData3
|
|
3364
|
+
| FacetData
|
|
3365
|
+
)
|
|
2777
3366
|
|
|
2778
3367
|
|
|
2779
3368
|
class Function(TypedDict):
|
|
@@ -2805,27 +3394,27 @@ class Function(TypedDict):
|
|
|
2805
3394
|
"""
|
|
2806
3395
|
Unique identifier for the prompt
|
|
2807
3396
|
"""
|
|
2808
|
-
description: NotRequired[
|
|
3397
|
+
description: NotRequired[str | None]
|
|
2809
3398
|
"""
|
|
2810
3399
|
Textual description of the prompt
|
|
2811
3400
|
"""
|
|
2812
|
-
created: NotRequired[
|
|
3401
|
+
created: NotRequired[str | None]
|
|
2813
3402
|
"""
|
|
2814
3403
|
Date of prompt creation
|
|
2815
3404
|
"""
|
|
2816
|
-
prompt_data: NotRequired[
|
|
2817
|
-
tags: NotRequired[
|
|
3405
|
+
prompt_data: NotRequired[PromptDataNullish | None]
|
|
3406
|
+
tags: NotRequired[Sequence[str] | None]
|
|
2818
3407
|
"""
|
|
2819
3408
|
A list of tags for the prompt
|
|
2820
3409
|
"""
|
|
2821
|
-
metadata: NotRequired[
|
|
3410
|
+
metadata: NotRequired[Mapping[str, Any] | None]
|
|
2822
3411
|
"""
|
|
2823
3412
|
User-controlled metadata about the prompt
|
|
2824
3413
|
"""
|
|
2825
|
-
function_type: NotRequired[
|
|
3414
|
+
function_type: NotRequired[FunctionTypeEnumNullish | None]
|
|
2826
3415
|
function_data: FunctionData
|
|
2827
|
-
origin: NotRequired[
|
|
2828
|
-
function_schema: NotRequired[
|
|
3416
|
+
origin: NotRequired[FunctionOrigin | None]
|
|
3417
|
+
function_schema: NotRequired[FunctionFunctionSchema | None]
|
|
2829
3418
|
"""
|
|
2830
3419
|
JSON schema for the function's parameters and return type
|
|
2831
3420
|
"""
|