braintrust 0.3.13__py3-none-any.whl → 0.3.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- braintrust/__init__.py +4 -0
- braintrust/_generated_types.py +596 -72
- braintrust/conftest.py +1 -0
- braintrust/functions/invoke.py +35 -2
- braintrust/generated_types.py +15 -1
- braintrust/gitutil.py +4 -0
- braintrust/logger.py +1 -1
- braintrust/oai.py +88 -6
- braintrust/score.py +1 -0
- braintrust/test_score.py +157 -0
- braintrust/version.py +2 -2
- braintrust/wrappers/pydantic_ai.py +1203 -0
- braintrust/wrappers/test_oai_attachments.py +322 -0
- braintrust/wrappers/test_pydantic_ai_integration.py +1788 -0
- braintrust/wrappers/{test_pydantic_ai.py → test_pydantic_ai_wrap_openai.py} +1 -2
- {braintrust-0.3.13.dist-info → braintrust-0.3.15.dist-info}/METADATA +1 -1
- {braintrust-0.3.13.dist-info → braintrust-0.3.15.dist-info}/RECORD +20 -16
- {braintrust-0.3.13.dist-info → braintrust-0.3.15.dist-info}/WHEEL +0 -0
- {braintrust-0.3.13.dist-info → braintrust-0.3.15.dist-info}/entry_points.txt +0 -0
- {braintrust-0.3.13.dist-info → braintrust-0.3.15.dist-info}/top_level.txt +0 -0
braintrust/_generated_types.py
CHANGED
|
@@ -8,9 +8,9 @@ from __future__ import annotations
|
|
|
8
8
|
|
|
9
9
|
from typing import Any, Literal, Mapping, Optional, Sequence, TypedDict, Union
|
|
10
10
|
|
|
11
|
-
from typing_extensions import NotRequired
|
|
11
|
+
from typing_extensions import NotRequired, TypeAlias
|
|
12
12
|
|
|
13
|
-
AclObjectType = Literal[
|
|
13
|
+
AclObjectType: TypeAlias = Literal[
|
|
14
14
|
'organization',
|
|
15
15
|
'project',
|
|
16
16
|
'experiment',
|
|
@@ -23,6 +23,9 @@ AclObjectType = Literal[
|
|
|
23
23
|
'project_log',
|
|
24
24
|
'org_project',
|
|
25
25
|
]
|
|
26
|
+
"""
|
|
27
|
+
The object type that the ACL applies to
|
|
28
|
+
"""
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
class AISecret(TypedDict):
|
|
@@ -116,7 +119,7 @@ class AsyncScoringControlAsyncScoringControl3(TypedDict):
|
|
|
116
119
|
class AsyncScoringStateAsyncScoringState(TypedDict):
|
|
117
120
|
status: Literal['enabled']
|
|
118
121
|
token: str
|
|
119
|
-
function_ids: Sequence
|
|
122
|
+
function_ids: Sequence[Any]
|
|
120
123
|
skip_logging: NotRequired[Optional[bool]]
|
|
121
124
|
|
|
122
125
|
|
|
@@ -124,7 +127,7 @@ class AsyncScoringStateAsyncScoringState1(TypedDict):
|
|
|
124
127
|
status: Literal['disabled']
|
|
125
128
|
|
|
126
129
|
|
|
127
|
-
AsyncScoringState = Optional[Union[AsyncScoringStateAsyncScoringState, AsyncScoringStateAsyncScoringState1]]
|
|
130
|
+
AsyncScoringState: TypeAlias = Optional[Union[AsyncScoringStateAsyncScoringState, AsyncScoringStateAsyncScoringState1]]
|
|
128
131
|
|
|
129
132
|
|
|
130
133
|
class BraintrustAttachmentReference(TypedDict):
|
|
@@ -200,7 +203,7 @@ class CallEventCallEvent7(TypedDict):
|
|
|
200
203
|
data: Literal['']
|
|
201
204
|
|
|
202
205
|
|
|
203
|
-
CallEvent = Union[
|
|
206
|
+
CallEvent: TypeAlias = Union[
|
|
204
207
|
CallEventCallEvent,
|
|
205
208
|
CallEventCallEvent1,
|
|
206
209
|
CallEventCallEvent2,
|
|
@@ -353,7 +356,7 @@ class ChatCompletionTool(TypedDict):
|
|
|
353
356
|
|
|
354
357
|
|
|
355
358
|
class CodeBundleRuntimeContext(TypedDict):
|
|
356
|
-
runtime: Literal['node', 'python', 'browser']
|
|
359
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
357
360
|
version: str
|
|
358
361
|
|
|
359
362
|
|
|
@@ -454,6 +457,18 @@ class EnvVar(TypedDict):
|
|
|
454
457
|
"""
|
|
455
458
|
Date the environment variable was last used
|
|
456
459
|
"""
|
|
460
|
+
metadata: NotRequired[Optional[Mapping[str, Any]]]
|
|
461
|
+
"""
|
|
462
|
+
Optional metadata associated with the environment variable when managed via the function secrets API
|
|
463
|
+
"""
|
|
464
|
+
secret_type: NotRequired[Optional[str]]
|
|
465
|
+
"""
|
|
466
|
+
Optional classification for the secret (for example, the AI provider name)
|
|
467
|
+
"""
|
|
468
|
+
secret_category: NotRequired[Optional[Literal['env_var', 'ai_provider']]]
|
|
469
|
+
"""
|
|
470
|
+
The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys
|
|
471
|
+
"""
|
|
457
472
|
|
|
458
473
|
|
|
459
474
|
class ExperimentEventMetadata(TypedDict):
|
|
@@ -529,7 +544,7 @@ class ExtendedSavedFunctionIdExtendedSavedFunctionId2(TypedDict):
|
|
|
529
544
|
slug: str
|
|
530
545
|
|
|
531
546
|
|
|
532
|
-
ExtendedSavedFunctionId = Union[
|
|
547
|
+
ExtendedSavedFunctionId: TypeAlias = Union[
|
|
533
548
|
ExtendedSavedFunctionIdExtendedSavedFunctionId,
|
|
534
549
|
ExtendedSavedFunctionIdExtendedSavedFunctionId1,
|
|
535
550
|
ExtendedSavedFunctionIdExtendedSavedFunctionId2,
|
|
@@ -555,6 +570,48 @@ class ExternalAttachmentReference(TypedDict):
|
|
|
555
570
|
"""
|
|
556
571
|
|
|
557
572
|
|
|
573
|
+
class PreprocessorPreprocessor(TypedDict):
|
|
574
|
+
type: Literal['function']
|
|
575
|
+
id: str
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
class PreprocessorPreprocessor1(TypedDict):
|
|
579
|
+
type: Literal['global']
|
|
580
|
+
name: str
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
class PreprocessorPreprocessor2(TypedDict):
|
|
584
|
+
pass
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
class PreprocessorPreprocessor3(PreprocessorPreprocessor, PreprocessorPreprocessor2):
|
|
588
|
+
pass
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
class PreprocessorPreprocessor4(PreprocessorPreprocessor1, PreprocessorPreprocessor2):
|
|
592
|
+
pass
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
Preprocessor: TypeAlias = Union[PreprocessorPreprocessor3, PreprocessorPreprocessor4]
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
class FacetData(TypedDict):
|
|
599
|
+
type: Literal['facet']
|
|
600
|
+
preprocessor: NotRequired[Optional[Preprocessor]]
|
|
601
|
+
prompt: str
|
|
602
|
+
"""
|
|
603
|
+
The prompt to use for LLM extraction. The preprocessed text will be provided as context.
|
|
604
|
+
"""
|
|
605
|
+
model: NotRequired[Optional[str]]
|
|
606
|
+
"""
|
|
607
|
+
The model to use for facet extraction
|
|
608
|
+
"""
|
|
609
|
+
no_match_pattern: NotRequired[Optional[str]]
|
|
610
|
+
"""
|
|
611
|
+
Regex pattern to identify outputs that do not match the facet. If the output matches, the facet will be saved as 'no_match'
|
|
612
|
+
"""
|
|
613
|
+
|
|
614
|
+
|
|
558
615
|
class FunctionOrigin(TypedDict):
|
|
559
616
|
object_type: Optional[AclObjectType]
|
|
560
617
|
object_id: str
|
|
@@ -581,7 +638,7 @@ class Data(CodeBundle):
|
|
|
581
638
|
|
|
582
639
|
|
|
583
640
|
class FunctionDataFunctionData1DataRuntimeContext(TypedDict):
|
|
584
|
-
runtime: Literal['node', 'python', 'browser']
|
|
641
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
585
642
|
version: str
|
|
586
643
|
|
|
587
644
|
|
|
@@ -589,6 +646,10 @@ class FunctionDataFunctionData1Data(TypedDict):
|
|
|
589
646
|
type: Literal['inline']
|
|
590
647
|
runtime_context: FunctionDataFunctionData1DataRuntimeContext
|
|
591
648
|
code: str
|
|
649
|
+
code_hash: NotRequired[Optional[str]]
|
|
650
|
+
"""
|
|
651
|
+
SHA256 hash of the code, computed at save time
|
|
652
|
+
"""
|
|
592
653
|
|
|
593
654
|
|
|
594
655
|
class FunctionDataFunctionData1(TypedDict):
|
|
@@ -606,9 +667,13 @@ class FunctionDataFunctionData2(TypedDict):
|
|
|
606
667
|
class FunctionDataFunctionData3(TypedDict):
|
|
607
668
|
type: Literal['global']
|
|
608
669
|
name: str
|
|
670
|
+
config: NotRequired[Optional[Mapping[str, Any]]]
|
|
671
|
+
"""
|
|
672
|
+
Configuration options to pass to the global function (e.g., for preprocessor customization)
|
|
673
|
+
"""
|
|
609
674
|
|
|
610
675
|
|
|
611
|
-
FunctionFormat = Literal['llm', 'code', 'global', 'graph']
|
|
676
|
+
FunctionFormat: TypeAlias = Literal['llm', 'code', 'global', 'graph']
|
|
612
677
|
|
|
613
678
|
|
|
614
679
|
class FunctionIdFunctionId(TypedDict):
|
|
@@ -660,7 +725,7 @@ class FunctionIdFunctionId3(TypedDict):
|
|
|
660
725
|
|
|
661
726
|
|
|
662
727
|
class FunctionIdFunctionId4InlineContext(TypedDict):
|
|
663
|
-
runtime: Literal['node', 'python', 'browser']
|
|
728
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
664
729
|
version: str
|
|
665
730
|
|
|
666
731
|
|
|
@@ -676,19 +741,21 @@ class FunctionIdFunctionId4(TypedDict):
|
|
|
676
741
|
"""
|
|
677
742
|
|
|
678
743
|
|
|
679
|
-
FunctionIdRef = Mapping[str, Any]
|
|
744
|
+
FunctionIdRef: TypeAlias = Mapping[str, Any]
|
|
680
745
|
|
|
681
746
|
|
|
682
|
-
FunctionObjectType = Literal[
|
|
747
|
+
FunctionObjectType: TypeAlias = Literal[
|
|
748
|
+
'prompt', 'tool', 'scorer', 'task', 'agent', 'custom_view', 'preprocessor', 'facet'
|
|
749
|
+
]
|
|
683
750
|
|
|
684
751
|
|
|
685
|
-
FunctionOutputType = Literal['completion', 'score', 'any']
|
|
752
|
+
FunctionOutputType: TypeAlias = Literal['completion', 'score', 'any']
|
|
686
753
|
|
|
687
754
|
|
|
688
|
-
FunctionTypeEnum = Literal['llm', 'scorer', 'task', 'tool', 'custom_view']
|
|
755
|
+
FunctionTypeEnum: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet']
|
|
689
756
|
|
|
690
757
|
|
|
691
|
-
FunctionTypeEnumNullish = Literal['llm', 'scorer', 'task', 'tool', 'custom_view']
|
|
758
|
+
FunctionTypeEnumNullish: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet']
|
|
692
759
|
|
|
693
760
|
|
|
694
761
|
class GitMetadataSettings(TypedDict):
|
|
@@ -969,7 +1036,79 @@ class Group(TypedDict):
|
|
|
969
1036
|
"""
|
|
970
1037
|
|
|
971
1038
|
|
|
972
|
-
IfExists = Literal['error', 'ignore', 'replace']
|
|
1039
|
+
IfExists: TypeAlias = Literal['error', 'ignore', 'replace']
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
class InvokeFunctionInvokeFunction(TypedDict):
|
|
1043
|
+
function_id: str
|
|
1044
|
+
"""
|
|
1045
|
+
The ID of the function
|
|
1046
|
+
"""
|
|
1047
|
+
version: NotRequired[Optional[str]]
|
|
1048
|
+
"""
|
|
1049
|
+
The version of the function
|
|
1050
|
+
"""
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
class InvokeFunctionInvokeFunction1(TypedDict):
|
|
1054
|
+
project_name: str
|
|
1055
|
+
"""
|
|
1056
|
+
The name of the project containing the function
|
|
1057
|
+
"""
|
|
1058
|
+
slug: str
|
|
1059
|
+
"""
|
|
1060
|
+
The slug of the function
|
|
1061
|
+
"""
|
|
1062
|
+
version: NotRequired[Optional[str]]
|
|
1063
|
+
"""
|
|
1064
|
+
The version of the function
|
|
1065
|
+
"""
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
class InvokeFunctionInvokeFunction2(TypedDict):
|
|
1069
|
+
global_function: str
|
|
1070
|
+
"""
|
|
1071
|
+
The name of the global function. Currently, the global namespace includes the functions in autoevals
|
|
1072
|
+
"""
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
class InvokeFunctionInvokeFunction3(TypedDict):
|
|
1076
|
+
prompt_session_id: str
|
|
1077
|
+
"""
|
|
1078
|
+
The ID of the prompt session
|
|
1079
|
+
"""
|
|
1080
|
+
prompt_session_function_id: str
|
|
1081
|
+
"""
|
|
1082
|
+
The ID of the function in the prompt session
|
|
1083
|
+
"""
|
|
1084
|
+
version: NotRequired[Optional[str]]
|
|
1085
|
+
"""
|
|
1086
|
+
The version of the function
|
|
1087
|
+
"""
|
|
1088
|
+
|
|
1089
|
+
|
|
1090
|
+
class InvokeFunctionInvokeFunction4InlineContext(TypedDict):
|
|
1091
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
1092
|
+
version: str
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
class InvokeFunctionInvokeFunction4(TypedDict):
|
|
1096
|
+
inline_context: InvokeFunctionInvokeFunction4InlineContext
|
|
1097
|
+
code: str
|
|
1098
|
+
"""
|
|
1099
|
+
The inline code to execute
|
|
1100
|
+
"""
|
|
1101
|
+
name: NotRequired[Optional[str]]
|
|
1102
|
+
"""
|
|
1103
|
+
The name of the inline code function
|
|
1104
|
+
"""
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
class InvokeFunctionMcpAuth(TypedDict):
|
|
1108
|
+
oauth_token: NotRequired[Optional[str]]
|
|
1109
|
+
"""
|
|
1110
|
+
The OAuth token to use
|
|
1111
|
+
"""
|
|
973
1112
|
|
|
974
1113
|
|
|
975
1114
|
class InvokeParentInvokeParentRowIds(TypedDict):
|
|
@@ -1003,10 +1142,48 @@ class InvokeParentInvokeParent(TypedDict):
|
|
|
1003
1142
|
"""
|
|
1004
1143
|
|
|
1005
1144
|
|
|
1006
|
-
InvokeParent = Union[InvokeParentInvokeParent, str]
|
|
1145
|
+
InvokeParent: TypeAlias = Union[InvokeParentInvokeParent, str]
|
|
1146
|
+
"""
|
|
1147
|
+
Options for tracing the function call
|
|
1148
|
+
"""
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
class MCPServer(TypedDict):
|
|
1152
|
+
id: str
|
|
1153
|
+
"""
|
|
1154
|
+
Unique identifier for the MCP server
|
|
1155
|
+
"""
|
|
1156
|
+
project_id: str
|
|
1157
|
+
"""
|
|
1158
|
+
Unique identifier for the project that the MCP server belongs under
|
|
1159
|
+
"""
|
|
1160
|
+
user_id: NotRequired[Optional[str]]
|
|
1161
|
+
"""
|
|
1162
|
+
Identifies the user who created the MCP server
|
|
1163
|
+
"""
|
|
1164
|
+
created: NotRequired[Optional[str]]
|
|
1165
|
+
"""
|
|
1166
|
+
Date of MCP server creation
|
|
1167
|
+
"""
|
|
1168
|
+
deleted_at: NotRequired[Optional[str]]
|
|
1169
|
+
"""
|
|
1170
|
+
Date of MCP server deletion, or null if the MCP server is still active
|
|
1171
|
+
"""
|
|
1172
|
+
name: str
|
|
1173
|
+
"""
|
|
1174
|
+
Name of the MCP server. Within a project, MCP server names are unique
|
|
1175
|
+
"""
|
|
1176
|
+
description: NotRequired[Optional[str]]
|
|
1177
|
+
"""
|
|
1178
|
+
Textual description of the MCP server
|
|
1179
|
+
"""
|
|
1180
|
+
url: str
|
|
1181
|
+
"""
|
|
1182
|
+
URL of the MCP server endpoint
|
|
1183
|
+
"""
|
|
1007
1184
|
|
|
1008
1185
|
|
|
1009
|
-
MessageRole = Literal['system', 'user', 'assistant', 'function', 'tool', 'model', 'developer']
|
|
1186
|
+
MessageRole: TypeAlias = Literal['system', 'user', 'assistant', 'function', 'tool', 'model', 'developer']
|
|
1010
1187
|
|
|
1011
1188
|
|
|
1012
1189
|
class ModelParamsModelParamsToolChoiceFunction(TypedDict):
|
|
@@ -1061,6 +1238,24 @@ class ModelParamsModelParams4(TypedDict):
|
|
|
1061
1238
|
reasoning_budget: NotRequired[Optional[float]]
|
|
1062
1239
|
|
|
1063
1240
|
|
|
1241
|
+
class NullableSavedFunctionIdNullableSavedFunctionId(TypedDict):
|
|
1242
|
+
type: Literal['function']
|
|
1243
|
+
id: str
|
|
1244
|
+
|
|
1245
|
+
|
|
1246
|
+
class NullableSavedFunctionIdNullableSavedFunctionId1(TypedDict):
|
|
1247
|
+
type: Literal['global']
|
|
1248
|
+
name: str
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
NullableSavedFunctionId: TypeAlias = Optional[
|
|
1252
|
+
Union[NullableSavedFunctionIdNullableSavedFunctionId, NullableSavedFunctionIdNullableSavedFunctionId1]
|
|
1253
|
+
]
|
|
1254
|
+
"""
|
|
1255
|
+
Default preprocessor for this project. When set, functions that use preprocessors will use this instead of their built-in default.
|
|
1256
|
+
"""
|
|
1257
|
+
|
|
1258
|
+
|
|
1064
1259
|
class ObjectReference(TypedDict):
|
|
1065
1260
|
object_type: Literal['project_logs', 'experiment', 'dataset', 'prompt', 'function', 'prompt_session']
|
|
1066
1261
|
"""
|
|
@@ -1126,7 +1321,14 @@ class Organization(TypedDict):
|
|
|
1126
1321
|
"""
|
|
1127
1322
|
|
|
1128
1323
|
|
|
1129
|
-
Permission = Literal[
|
|
1324
|
+
Permission: TypeAlias = Literal[
|
|
1325
|
+
'create', 'read', 'update', 'delete', 'create_acls', 'read_acls', 'update_acls', 'delete_acls'
|
|
1326
|
+
]
|
|
1327
|
+
"""
|
|
1328
|
+
Each permission permits a certain type of operation on an object in the system
|
|
1329
|
+
|
|
1330
|
+
Permissions can be assigned to to objects on an individual basis, or grouped into roles
|
|
1331
|
+
"""
|
|
1130
1332
|
|
|
1131
1333
|
|
|
1132
1334
|
class ProjectAutomationConfigAction(TypedDict):
|
|
@@ -1140,6 +1342,25 @@ class ProjectAutomationConfigAction(TypedDict):
|
|
|
1140
1342
|
"""
|
|
1141
1343
|
|
|
1142
1344
|
|
|
1345
|
+
class ProjectAutomationConfigAction1(TypedDict):
|
|
1346
|
+
type: Literal['slack']
|
|
1347
|
+
"""
|
|
1348
|
+
The type of action to take
|
|
1349
|
+
"""
|
|
1350
|
+
workspace_id: str
|
|
1351
|
+
"""
|
|
1352
|
+
The Slack workspace ID to post to
|
|
1353
|
+
"""
|
|
1354
|
+
channel: str
|
|
1355
|
+
"""
|
|
1356
|
+
The Slack channel ID to post to
|
|
1357
|
+
"""
|
|
1358
|
+
message_template: NotRequired[Optional[str]]
|
|
1359
|
+
"""
|
|
1360
|
+
Custom message template for the alert
|
|
1361
|
+
"""
|
|
1362
|
+
|
|
1363
|
+
|
|
1143
1364
|
class ProjectAutomationConfig(TypedDict):
|
|
1144
1365
|
event_type: Literal['logs']
|
|
1145
1366
|
"""
|
|
@@ -1153,7 +1374,7 @@ class ProjectAutomationConfig(TypedDict):
|
|
|
1153
1374
|
"""
|
|
1154
1375
|
Perform the triggered action at most once in this interval of seconds
|
|
1155
1376
|
"""
|
|
1156
|
-
action: ProjectAutomationConfigAction
|
|
1377
|
+
action: Union[ProjectAutomationConfigAction, ProjectAutomationConfigAction1]
|
|
1157
1378
|
"""
|
|
1158
1379
|
The action to take when the automation rule is triggered
|
|
1159
1380
|
"""
|
|
@@ -1287,7 +1508,10 @@ class ProjectScoreCategory(TypedDict):
|
|
|
1287
1508
|
"""
|
|
1288
1509
|
|
|
1289
1510
|
|
|
1290
|
-
ProjectScoreType = Literal['slider', 'categorical', 'weighted', 'minimum', 'maximum', 'online', 'free-form']
|
|
1511
|
+
ProjectScoreType: TypeAlias = Literal['slider', 'categorical', 'weighted', 'minimum', 'maximum', 'online', 'free-form']
|
|
1512
|
+
"""
|
|
1513
|
+
The type of the configured score
|
|
1514
|
+
"""
|
|
1291
1515
|
|
|
1292
1516
|
|
|
1293
1517
|
class ProjectSettingsSpanFieldOrderItem(TypedDict):
|
|
@@ -1324,6 +1548,7 @@ class ProjectSettings(TypedDict):
|
|
|
1324
1548
|
"""
|
|
1325
1549
|
If true, disable real-time queries for this project. This can improve query performance for high-volume logs.
|
|
1326
1550
|
"""
|
|
1551
|
+
default_preprocessor: NotRequired[Optional[NullableSavedFunctionId]]
|
|
1327
1552
|
|
|
1328
1553
|
|
|
1329
1554
|
class ProjectTag(TypedDict):
|
|
@@ -1505,7 +1730,7 @@ class ResponseFormatNullishResponseFormatNullish2(TypedDict):
|
|
|
1505
1730
|
type: Literal['text']
|
|
1506
1731
|
|
|
1507
1732
|
|
|
1508
|
-
ResponseFormatNullish = Optional[
|
|
1733
|
+
ResponseFormatNullish: TypeAlias = Optional[
|
|
1509
1734
|
Union[
|
|
1510
1735
|
ResponseFormatNullishResponseFormatNullish,
|
|
1511
1736
|
ResponseFormatNullishResponseFormatNullish1,
|
|
@@ -1514,7 +1739,10 @@ ResponseFormatNullish = Optional[
|
|
|
1514
1739
|
]
|
|
1515
1740
|
|
|
1516
1741
|
|
|
1517
|
-
RetentionObjectType = Literal['project_logs', 'experiment', 'dataset']
|
|
1742
|
+
RetentionObjectType: TypeAlias = Literal['project_logs', 'experiment', 'dataset']
|
|
1743
|
+
"""
|
|
1744
|
+
The object type that the retention policy applies to
|
|
1745
|
+
"""
|
|
1518
1746
|
|
|
1519
1747
|
|
|
1520
1748
|
class RoleMemberPermission(TypedDict):
|
|
@@ -1579,7 +1807,145 @@ class RunEvalData1(TypedDict):
|
|
|
1579
1807
|
|
|
1580
1808
|
|
|
1581
1809
|
class RunEvalData2(TypedDict):
|
|
1582
|
-
data: Sequence
|
|
1810
|
+
data: Sequence[Any]
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
class TaskTask(TypedDict):
|
|
1814
|
+
function_id: str
|
|
1815
|
+
"""
|
|
1816
|
+
The ID of the function
|
|
1817
|
+
"""
|
|
1818
|
+
version: NotRequired[Optional[str]]
|
|
1819
|
+
"""
|
|
1820
|
+
The version of the function
|
|
1821
|
+
"""
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
class TaskTask1(TypedDict):
|
|
1825
|
+
project_name: str
|
|
1826
|
+
"""
|
|
1827
|
+
The name of the project containing the function
|
|
1828
|
+
"""
|
|
1829
|
+
slug: str
|
|
1830
|
+
"""
|
|
1831
|
+
The slug of the function
|
|
1832
|
+
"""
|
|
1833
|
+
version: NotRequired[Optional[str]]
|
|
1834
|
+
"""
|
|
1835
|
+
The version of the function
|
|
1836
|
+
"""
|
|
1837
|
+
|
|
1838
|
+
|
|
1839
|
+
class TaskTask2(TypedDict):
|
|
1840
|
+
global_function: str
|
|
1841
|
+
"""
|
|
1842
|
+
The name of the global function. Currently, the global namespace includes the functions in autoevals
|
|
1843
|
+
"""
|
|
1844
|
+
|
|
1845
|
+
|
|
1846
|
+
class TaskTask3(TypedDict):
|
|
1847
|
+
prompt_session_id: str
|
|
1848
|
+
"""
|
|
1849
|
+
The ID of the prompt session
|
|
1850
|
+
"""
|
|
1851
|
+
prompt_session_function_id: str
|
|
1852
|
+
"""
|
|
1853
|
+
The ID of the function in the prompt session
|
|
1854
|
+
"""
|
|
1855
|
+
version: NotRequired[Optional[str]]
|
|
1856
|
+
"""
|
|
1857
|
+
The version of the function
|
|
1858
|
+
"""
|
|
1859
|
+
|
|
1860
|
+
|
|
1861
|
+
class TaskTask4InlineContext(TypedDict):
|
|
1862
|
+
runtime: Literal['node', 'python', 'browser', 'quickjs']
|
|
1863
|
+
version: str
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
class TaskTask4(TypedDict):
|
|
1867
|
+
inline_context: TaskTask4InlineContext
|
|
1868
|
+
code: str
|
|
1869
|
+
"""
|
|
1870
|
+
The inline code to execute
|
|
1871
|
+
"""
|
|
1872
|
+
name: NotRequired[Optional[str]]
|
|
1873
|
+
"""
|
|
1874
|
+
The name of the inline code function
|
|
1875
|
+
"""
|
|
1876
|
+
|
|
1877
|
+
|
|
1878
|
+
class TaskTask7(TypedDict):
|
|
1879
|
+
pass
|
|
1880
|
+
|
|
1881
|
+
|
|
1882
|
+
class TaskTask8(TaskTask, TaskTask7):
|
|
1883
|
+
pass
|
|
1884
|
+
|
|
1885
|
+
|
|
1886
|
+
class TaskTask9(TaskTask1, TaskTask7):
|
|
1887
|
+
pass
|
|
1888
|
+
|
|
1889
|
+
|
|
1890
|
+
class TaskTask10(TaskTask2, TaskTask7):
|
|
1891
|
+
pass
|
|
1892
|
+
|
|
1893
|
+
|
|
1894
|
+
class TaskTask11(TaskTask3, TaskTask7):
|
|
1895
|
+
pass
|
|
1896
|
+
|
|
1897
|
+
|
|
1898
|
+
class TaskTask12(TaskTask4, TaskTask7):
|
|
1899
|
+
pass
|
|
1900
|
+
|
|
1901
|
+
|
|
1902
|
+
class ParentParentRowIds(TypedDict):
|
|
1903
|
+
id: str
|
|
1904
|
+
"""
|
|
1905
|
+
The id of the row
|
|
1906
|
+
"""
|
|
1907
|
+
span_id: str
|
|
1908
|
+
"""
|
|
1909
|
+
The span_id of the row
|
|
1910
|
+
"""
|
|
1911
|
+
root_span_id: str
|
|
1912
|
+
"""
|
|
1913
|
+
The root_span_id of the row
|
|
1914
|
+
"""
|
|
1915
|
+
|
|
1916
|
+
|
|
1917
|
+
class ParentParent(TypedDict):
|
|
1918
|
+
object_type: Literal['project_logs', 'experiment', 'playground_logs']
|
|
1919
|
+
object_id: str
|
|
1920
|
+
"""
|
|
1921
|
+
The id of the container object you are logging to
|
|
1922
|
+
"""
|
|
1923
|
+
row_ids: NotRequired[Optional[ParentParentRowIds]]
|
|
1924
|
+
"""
|
|
1925
|
+
Identifiers for the row to to log a subspan under
|
|
1926
|
+
"""
|
|
1927
|
+
propagated_event: NotRequired[Optional[Mapping[str, Any]]]
|
|
1928
|
+
"""
|
|
1929
|
+
Include these properties in every span created under this parent
|
|
1930
|
+
"""
|
|
1931
|
+
|
|
1932
|
+
|
|
1933
|
+
class ParentParent1(TypedDict):
|
|
1934
|
+
pass
|
|
1935
|
+
|
|
1936
|
+
|
|
1937
|
+
class ParentParent2(ParentParent, ParentParent1):
|
|
1938
|
+
pass
|
|
1939
|
+
|
|
1940
|
+
|
|
1941
|
+
Parent: TypeAlias = ParentParent2
|
|
1942
|
+
|
|
1943
|
+
|
|
1944
|
+
class RunEvalMcpAuth(TypedDict):
|
|
1945
|
+
oauth_token: NotRequired[Optional[str]]
|
|
1946
|
+
"""
|
|
1947
|
+
The OAuth token to use
|
|
1948
|
+
"""
|
|
1583
1949
|
|
|
1584
1950
|
|
|
1585
1951
|
class SavedFunctionIdSavedFunctionId(TypedDict):
|
|
@@ -1592,7 +1958,7 @@ class SavedFunctionIdSavedFunctionId1(TypedDict):
|
|
|
1592
1958
|
name: str
|
|
1593
1959
|
|
|
1594
1960
|
|
|
1595
|
-
SavedFunctionId = Union[SavedFunctionIdSavedFunctionId, SavedFunctionIdSavedFunctionId1]
|
|
1961
|
+
SavedFunctionId: TypeAlias = Union[SavedFunctionIdSavedFunctionId, SavedFunctionIdSavedFunctionId1]
|
|
1596
1962
|
|
|
1597
1963
|
|
|
1598
1964
|
class ServiceToken(TypedDict):
|
|
@@ -1666,7 +2032,22 @@ class SpanIFrame(TypedDict):
|
|
|
1666
2032
|
"""
|
|
1667
2033
|
|
|
1668
2034
|
|
|
1669
|
-
|
|
2035
|
+
class SpanScope(TypedDict):
|
|
2036
|
+
type: Literal['span']
|
|
2037
|
+
root_span_id: str
|
|
2038
|
+
"""
|
|
2039
|
+
The root span id is a unique identifier for the trace.
|
|
2040
|
+
"""
|
|
2041
|
+
id: str
|
|
2042
|
+
"""
|
|
2043
|
+
A unique identifier for the span.
|
|
2044
|
+
"""
|
|
2045
|
+
|
|
2046
|
+
|
|
2047
|
+
SpanType: TypeAlias = Literal['llm', 'score', 'function', 'eval', 'task', 'tool']
|
|
2048
|
+
"""
|
|
2049
|
+
Type of the span, for display purposes only
|
|
2050
|
+
"""
|
|
1670
2051
|
|
|
1671
2052
|
|
|
1672
2053
|
class SSEConsoleEventData(TypedDict):
|
|
@@ -1688,7 +2069,10 @@ class SSEProgressEventData(TypedDict):
|
|
|
1688
2069
|
data: str
|
|
1689
2070
|
|
|
1690
2071
|
|
|
1691
|
-
StreamingMode = Literal['auto', 'parallel']
|
|
2072
|
+
StreamingMode: TypeAlias = Literal['auto', 'parallel']
|
|
2073
|
+
"""
|
|
2074
|
+
The mode format of the returned value (defaults to 'auto')
|
|
2075
|
+
"""
|
|
1692
2076
|
|
|
1693
2077
|
|
|
1694
2078
|
class ToolFunctionDefinitionFunction(TypedDict):
|
|
@@ -1703,7 +2087,15 @@ class ToolFunctionDefinition(TypedDict):
|
|
|
1703
2087
|
function: ToolFunctionDefinitionFunction
|
|
1704
2088
|
|
|
1705
2089
|
|
|
1706
|
-
|
|
2090
|
+
class TraceScope(TypedDict):
|
|
2091
|
+
type: Literal['trace']
|
|
2092
|
+
root_span_id: str
|
|
2093
|
+
"""
|
|
2094
|
+
The root span id is a unique identifier for the trace.
|
|
2095
|
+
"""
|
|
2096
|
+
|
|
2097
|
+
|
|
2098
|
+
UploadStatus: TypeAlias = Literal['uploading', 'done', 'error']
|
|
1707
2099
|
|
|
1708
2100
|
|
|
1709
2101
|
class User(TypedDict):
|
|
@@ -1755,6 +2147,7 @@ class ViewOptionsViewOptionsOptions(TypedDict):
|
|
|
1755
2147
|
class ViewOptionsViewOptions(TypedDict):
|
|
1756
2148
|
viewType: Literal['monitor']
|
|
1757
2149
|
options: ViewOptionsViewOptionsOptions
|
|
2150
|
+
freezeColumns: NotRequired[Optional[bool]]
|
|
1758
2151
|
|
|
1759
2152
|
|
|
1760
2153
|
class ViewOptionsViewOptions1ExcludedMeasure(TypedDict):
|
|
@@ -1811,9 +2204,13 @@ class ViewOptionsViewOptions1(TypedDict):
|
|
|
1811
2204
|
chartAnnotations: NotRequired[Optional[Sequence[ViewOptionsViewOptions1ChartAnnotation]]]
|
|
1812
2205
|
timeRangeFilter: NotRequired[Optional[Union[str, ViewOptionsViewOptions1TimeRangeFilter]]]
|
|
1813
2206
|
queryShape: NotRequired[Optional[Literal['traces', 'spans']]]
|
|
2207
|
+
freezeColumns: NotRequired[Optional[bool]]
|
|
1814
2208
|
|
|
1815
2209
|
|
|
1816
|
-
ViewOptions = Optional[Union[ViewOptionsViewOptions, ViewOptionsViewOptions1]]
|
|
2210
|
+
ViewOptions: TypeAlias = Optional[Union[ViewOptionsViewOptions, ViewOptionsViewOptions1]]
|
|
2211
|
+
"""
|
|
2212
|
+
Options for the view in the app
|
|
2213
|
+
"""
|
|
1817
2214
|
|
|
1818
2215
|
|
|
1819
2216
|
class Acl(TypedDict):
|
|
@@ -1883,10 +2280,10 @@ class AnyModelParams(TypedDict):
|
|
|
1883
2280
|
|
|
1884
2281
|
class AsyncScoringControlAsyncScoringControl1(TypedDict):
|
|
1885
2282
|
kind: Literal['state_override']
|
|
1886
|
-
state: AsyncScoringState
|
|
2283
|
+
state: Optional[AsyncScoringState]
|
|
1887
2284
|
|
|
1888
2285
|
|
|
1889
|
-
AsyncScoringControl = Union[
|
|
2286
|
+
AsyncScoringControl: TypeAlias = Union[
|
|
1890
2287
|
AsyncScoringControlAsyncScoringControl,
|
|
1891
2288
|
AsyncScoringControlAsyncScoringControl1,
|
|
1892
2289
|
AsyncScoringControlAsyncScoringControl2,
|
|
@@ -1894,7 +2291,7 @@ AsyncScoringControl = Union[
|
|
|
1894
2291
|
]
|
|
1895
2292
|
|
|
1896
2293
|
|
|
1897
|
-
AttachmentReference = Union[BraintrustAttachmentReference, ExternalAttachmentReference]
|
|
2294
|
+
AttachmentReference: TypeAlias = Union[BraintrustAttachmentReference, ExternalAttachmentReference]
|
|
1898
2295
|
|
|
1899
2296
|
|
|
1900
2297
|
class AttachmentStatus(TypedDict):
|
|
@@ -1905,7 +2302,7 @@ class AttachmentStatus(TypedDict):
|
|
|
1905
2302
|
"""
|
|
1906
2303
|
|
|
1907
2304
|
|
|
1908
|
-
ChatCompletionContentPart = Union[
|
|
2305
|
+
ChatCompletionContentPart: TypeAlias = Union[
|
|
1909
2306
|
ChatCompletionContentPartTextWithTitle,
|
|
1910
2307
|
ChatCompletionContentPartImageWithTitle,
|
|
1911
2308
|
ChatCompletionContentPartFileWithTitle,
|
|
@@ -1927,7 +2324,7 @@ class ChatCompletionMessageParamChatCompletionMessageParam2(TypedDict):
|
|
|
1927
2324
|
reasoning: NotRequired[Optional[Sequence[ChatCompletionMessageReasoning]]]
|
|
1928
2325
|
|
|
1929
2326
|
|
|
1930
|
-
ChatCompletionMessageParam = Union[
|
|
2327
|
+
ChatCompletionMessageParam: TypeAlias = Union[
|
|
1931
2328
|
ChatCompletionMessageParamChatCompletionMessageParam,
|
|
1932
2329
|
ChatCompletionMessageParamChatCompletionMessageParam1,
|
|
1933
2330
|
ChatCompletionMessageParamChatCompletionMessageParam2,
|
|
@@ -1944,7 +2341,7 @@ class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1(TypedDic
|
|
|
1944
2341
|
name: NotRequired[Optional[str]]
|
|
1945
2342
|
|
|
1946
2343
|
|
|
1947
|
-
ChatCompletionOpenAIMessageParam = Union[
|
|
2344
|
+
ChatCompletionOpenAIMessageParam: TypeAlias = Union[
|
|
1948
2345
|
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam,
|
|
1949
2346
|
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1,
|
|
1950
2347
|
ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2,
|
|
@@ -2078,6 +2475,12 @@ class Experiment(TypedDict):
|
|
|
2078
2475
|
"""
|
|
2079
2476
|
|
|
2080
2477
|
|
|
2478
|
+
InvokeScope: TypeAlias = Union[SpanScope, TraceScope]
|
|
2479
|
+
"""
|
|
2480
|
+
The scope at which to operate (span or trace)
|
|
2481
|
+
"""
|
|
2482
|
+
|
|
2483
|
+
|
|
2081
2484
|
class ModelParamsModelParams(TypedDict):
|
|
2082
2485
|
use_cache: NotRequired[Optional[bool]]
|
|
2083
2486
|
reasoning_enabled: NotRequired[Optional[bool]]
|
|
@@ -2102,7 +2505,7 @@ class ModelParamsModelParams(TypedDict):
|
|
|
2102
2505
|
verbosity: NotRequired[Optional[Literal['low', 'medium', 'high']]]
|
|
2103
2506
|
|
|
2104
2507
|
|
|
2105
|
-
ModelParams = Union[
|
|
2508
|
+
ModelParams: TypeAlias = Union[
|
|
2106
2509
|
ModelParamsModelParams,
|
|
2107
2510
|
ModelParamsModelParams1,
|
|
2108
2511
|
ModelParamsModelParams2,
|
|
@@ -2209,7 +2612,7 @@ class ProjectAutomation(TypedDict):
|
|
|
2209
2612
|
"""
|
|
2210
2613
|
|
|
2211
2614
|
|
|
2212
|
-
ProjectScoreCategories = Optional[Union[Sequence[ProjectScoreCategory], Mapping[str, float], Sequence[str]]]
|
|
2615
|
+
ProjectScoreCategories: TypeAlias = Optional[Union[Sequence[ProjectScoreCategory], Mapping[str, float], Sequence[str]]]
|
|
2213
2616
|
|
|
2214
2617
|
|
|
2215
2618
|
class ProjectScoreConfig(TypedDict):
|
|
@@ -2224,7 +2627,7 @@ class PromptBlockDataPromptBlockData(TypedDict):
|
|
|
2224
2627
|
tools: NotRequired[Optional[str]]
|
|
2225
2628
|
|
|
2226
2629
|
|
|
2227
|
-
PromptBlockData = Union[PromptBlockDataPromptBlockData, PromptBlockDataPromptBlockData1]
|
|
2630
|
+
PromptBlockData: TypeAlias = Union[PromptBlockDataPromptBlockData, PromptBlockDataPromptBlockData1]
|
|
2228
2631
|
|
|
2229
2632
|
|
|
2230
2633
|
class PromptBlockDataNullishPromptBlockDataNullish(TypedDict):
|
|
@@ -2233,7 +2636,7 @@ class PromptBlockDataNullishPromptBlockDataNullish(TypedDict):
|
|
|
2233
2636
|
tools: NotRequired[Optional[str]]
|
|
2234
2637
|
|
|
2235
2638
|
|
|
2236
|
-
PromptBlockDataNullish = Optional[
|
|
2639
|
+
PromptBlockDataNullish: TypeAlias = Optional[
|
|
2237
2640
|
Union[PromptBlockDataNullishPromptBlockDataNullish, PromptBlockDataNullishPromptBlockDataNullish1]
|
|
2238
2641
|
]
|
|
2239
2642
|
|
|
@@ -2255,7 +2658,9 @@ class ResponseFormatResponseFormat1(TypedDict):
|
|
|
2255
2658
|
json_schema: ResponseFormatJsonSchema
|
|
2256
2659
|
|
|
2257
2660
|
|
|
2258
|
-
ResponseFormat = Union[
|
|
2661
|
+
ResponseFormat: TypeAlias = Union[
|
|
2662
|
+
ResponseFormatResponseFormat, ResponseFormatResponseFormat1, ResponseFormatResponseFormat2
|
|
2663
|
+
]
|
|
2259
2664
|
|
|
2260
2665
|
|
|
2261
2666
|
class SpanAttributes(TypedDict):
|
|
@@ -2373,7 +2778,7 @@ class GraphNodeGraphNode7(TypedDict):
|
|
|
2373
2778
|
prompt: PromptBlockData
|
|
2374
2779
|
|
|
2375
2780
|
|
|
2376
|
-
GraphNode = Union[
|
|
2781
|
+
GraphNode: TypeAlias = Union[
|
|
2377
2782
|
GraphNodeGraphNode,
|
|
2378
2783
|
GraphNodeGraphNode1,
|
|
2379
2784
|
GraphNodeGraphNode2,
|
|
@@ -2385,6 +2790,80 @@ GraphNode = Union[
|
|
|
2385
2790
|
]
|
|
2386
2791
|
|
|
2387
2792
|
|
|
2793
|
+
class InvokeContext(TypedDict):
|
|
2794
|
+
object_type: Literal['project_logs', 'experiment', 'dataset', 'playground_logs']
|
|
2795
|
+
"""
|
|
2796
|
+
The type of object containing the span data
|
|
2797
|
+
"""
|
|
2798
|
+
object_id: str
|
|
2799
|
+
"""
|
|
2800
|
+
The ID of the object containing the span data
|
|
2801
|
+
"""
|
|
2802
|
+
scope: InvokeScope
|
|
2803
|
+
|
|
2804
|
+
|
|
2805
|
+
class InvokeFunctionInvokeFunction7(TypedDict):
|
|
2806
|
+
input: NotRequired[Optional[Any]]
|
|
2807
|
+
"""
|
|
2808
|
+
Argument to the function, which can be any JSON serializable value
|
|
2809
|
+
"""
|
|
2810
|
+
expected: NotRequired[Optional[Any]]
|
|
2811
|
+
"""
|
|
2812
|
+
The expected output of the function
|
|
2813
|
+
"""
|
|
2814
|
+
metadata: NotRequired[Optional[Mapping[str, Any]]]
|
|
2815
|
+
"""
|
|
2816
|
+
Any relevant metadata. This will be logged and available as the `metadata` argument.
|
|
2817
|
+
"""
|
|
2818
|
+
tags: NotRequired[Optional[Sequence[str]]]
|
|
2819
|
+
"""
|
|
2820
|
+
Any relevant tags to log on the span.
|
|
2821
|
+
"""
|
|
2822
|
+
messages: NotRequired[Optional[Sequence[ChatCompletionMessageParam]]]
|
|
2823
|
+
"""
|
|
2824
|
+
If the function is an LLM, additional messages to pass along to it
|
|
2825
|
+
"""
|
|
2826
|
+
context: NotRequired[Optional[InvokeContext]]
|
|
2827
|
+
parent: NotRequired[Optional[InvokeParent]]
|
|
2828
|
+
stream: NotRequired[Optional[bool]]
|
|
2829
|
+
"""
|
|
2830
|
+
Whether to stream the response. If true, results will be returned in the Braintrust SSE format.
|
|
2831
|
+
"""
|
|
2832
|
+
mode: NotRequired[Optional[StreamingMode]]
|
|
2833
|
+
strict: NotRequired[Optional[bool]]
|
|
2834
|
+
"""
|
|
2835
|
+
If true, throw an error if one of the variables in the prompt is not present in the input
|
|
2836
|
+
"""
|
|
2837
|
+
mcp_auth: NotRequired[Optional[Mapping[str, InvokeFunctionMcpAuth]]]
|
|
2838
|
+
"""
|
|
2839
|
+
Map of MCP server URL to auth credentials
|
|
2840
|
+
"""
|
|
2841
|
+
overrides: NotRequired[Optional[Mapping[str, Any]]]
|
|
2842
|
+
"""
|
|
2843
|
+
Partial function definition to merge with the function being invoked. Fields are validated against the function type's schema at runtime. For facets: { preprocessor?, prompt?, model? }. For prompts: { model?, ... }.
|
|
2844
|
+
"""
|
|
2845
|
+
|
|
2846
|
+
|
|
2847
|
+
class InvokeFunctionInvokeFunction8(InvokeFunctionInvokeFunction, InvokeFunctionInvokeFunction7):
|
|
2848
|
+
pass
|
|
2849
|
+
|
|
2850
|
+
|
|
2851
|
+
class InvokeFunctionInvokeFunction9(InvokeFunctionInvokeFunction1, InvokeFunctionInvokeFunction7):
|
|
2852
|
+
pass
|
|
2853
|
+
|
|
2854
|
+
|
|
2855
|
+
class InvokeFunctionInvokeFunction10(InvokeFunctionInvokeFunction2, InvokeFunctionInvokeFunction7):
|
|
2856
|
+
pass
|
|
2857
|
+
|
|
2858
|
+
|
|
2859
|
+
class InvokeFunctionInvokeFunction11(InvokeFunctionInvokeFunction3, InvokeFunctionInvokeFunction7):
|
|
2860
|
+
pass
|
|
2861
|
+
|
|
2862
|
+
|
|
2863
|
+
class InvokeFunctionInvokeFunction12(InvokeFunctionInvokeFunction4, InvokeFunctionInvokeFunction7):
|
|
2864
|
+
pass
|
|
2865
|
+
|
|
2866
|
+
|
|
2388
2867
|
class ProjectLogsEvent(TypedDict):
|
|
2389
2868
|
id: str
|
|
2390
2869
|
"""
|
|
@@ -2518,6 +2997,7 @@ class PromptData(TypedDict):
|
|
|
2518
2997
|
options: NotRequired[Optional[PromptOptionsNullish]]
|
|
2519
2998
|
parser: NotRequired[Optional[PromptParserNullish]]
|
|
2520
2999
|
tool_functions: NotRequired[Optional[Sequence[SavedFunctionId]]]
|
|
3000
|
+
mcp: NotRequired[Optional[Mapping[str, Any]]]
|
|
2521
3001
|
origin: NotRequired[Optional[PromptDataOrigin]]
|
|
2522
3002
|
|
|
2523
3003
|
|
|
@@ -2526,9 +3006,40 @@ class PromptDataNullish(TypedDict):
|
|
|
2526
3006
|
options: NotRequired[Optional[PromptOptionsNullish]]
|
|
2527
3007
|
parser: NotRequired[Optional[PromptParserNullish]]
|
|
2528
3008
|
tool_functions: NotRequired[Optional[Sequence[SavedFunctionId]]]
|
|
3009
|
+
mcp: NotRequired[Optional[Mapping[str, Any]]]
|
|
2529
3010
|
origin: NotRequired[Optional[PromptDataNullishOrigin]]
|
|
2530
3011
|
|
|
2531
3012
|
|
|
3013
|
+
class TaskTask5(TypedDict):
|
|
3014
|
+
inline_prompt: NotRequired[Optional[PromptData]]
|
|
3015
|
+
inline_function: Mapping[str, Any]
|
|
3016
|
+
function_type: NotRequired[Optional[FunctionTypeEnum]]
|
|
3017
|
+
name: NotRequired[Optional[str]]
|
|
3018
|
+
"""
|
|
3019
|
+
The name of the inline function
|
|
3020
|
+
"""
|
|
3021
|
+
|
|
3022
|
+
|
|
3023
|
+
class TaskTask6(TypedDict):
|
|
3024
|
+
inline_prompt: PromptData
|
|
3025
|
+
function_type: NotRequired[Optional[FunctionTypeEnum]]
|
|
3026
|
+
name: NotRequired[Optional[str]]
|
|
3027
|
+
"""
|
|
3028
|
+
The name of the inline prompt
|
|
3029
|
+
"""
|
|
3030
|
+
|
|
3031
|
+
|
|
3032
|
+
class TaskTask13(TaskTask5, TaskTask7):
|
|
3033
|
+
pass
|
|
3034
|
+
|
|
3035
|
+
|
|
3036
|
+
class TaskTask14(TaskTask6, TaskTask7):
|
|
3037
|
+
pass
|
|
3038
|
+
|
|
3039
|
+
|
|
3040
|
+
Task: TypeAlias = Union[TaskTask8, TaskTask9, TaskTask10, TaskTask11, TaskTask12, TaskTask13, TaskTask14]
|
|
3041
|
+
|
|
3042
|
+
|
|
2532
3043
|
class View(TypedDict):
|
|
2533
3044
|
id: str
|
|
2534
3045
|
"""
|
|
@@ -2597,7 +3108,7 @@ class FunctionIdFunctionId6(TypedDict):
|
|
|
2597
3108
|
"""
|
|
2598
3109
|
|
|
2599
3110
|
|
|
2600
|
-
FunctionId = Union[
|
|
3111
|
+
FunctionId: TypeAlias = Union[
|
|
2601
3112
|
FunctionIdFunctionId,
|
|
2602
3113
|
FunctionIdFunctionId1,
|
|
2603
3114
|
FunctionIdFunctionId2,
|
|
@@ -2606,6 +3117,9 @@ FunctionId = Union[
|
|
|
2606
3117
|
FunctionIdFunctionId5,
|
|
2607
3118
|
FunctionIdFunctionId6,
|
|
2608
3119
|
]
|
|
3120
|
+
"""
|
|
3121
|
+
Options for identifying a function
|
|
3122
|
+
"""
|
|
2609
3123
|
|
|
2610
3124
|
|
|
2611
3125
|
class GraphData(TypedDict):
|
|
@@ -2614,39 +3128,47 @@ class GraphData(TypedDict):
|
|
|
2614
3128
|
edges: Mapping[str, GraphEdge]
|
|
2615
3129
|
|
|
2616
3130
|
|
|
2617
|
-
class
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
expected: NotRequired[Optional[Any]]
|
|
2623
|
-
"""
|
|
2624
|
-
The expected output of the function
|
|
2625
|
-
"""
|
|
2626
|
-
metadata: NotRequired[Optional[Mapping[str, Any]]]
|
|
2627
|
-
"""
|
|
2628
|
-
Any relevant metadata. This will be logged and available as the `metadata` argument.
|
|
2629
|
-
"""
|
|
2630
|
-
tags: NotRequired[Optional[Sequence[str]]]
|
|
2631
|
-
"""
|
|
2632
|
-
Any relevant tags to log on the span.
|
|
2633
|
-
"""
|
|
2634
|
-
messages: NotRequired[Optional[Sequence[ChatCompletionMessageParam]]]
|
|
2635
|
-
"""
|
|
2636
|
-
If the function is an LLM, additional messages to pass along to it
|
|
2637
|
-
"""
|
|
2638
|
-
parent: NotRequired[Optional[InvokeParent]]
|
|
2639
|
-
stream: NotRequired[Optional[bool]]
|
|
3131
|
+
class InvokeFunctionInvokeFunction5(TypedDict):
|
|
3132
|
+
inline_prompt: NotRequired[Optional[PromptData]]
|
|
3133
|
+
inline_function: Mapping[str, Any]
|
|
3134
|
+
function_type: NotRequired[Optional[FunctionTypeEnum]]
|
|
3135
|
+
name: NotRequired[Optional[str]]
|
|
2640
3136
|
"""
|
|
2641
|
-
|
|
3137
|
+
The name of the inline function
|
|
2642
3138
|
"""
|
|
2643
|
-
|
|
2644
|
-
|
|
3139
|
+
|
|
3140
|
+
|
|
3141
|
+
class InvokeFunctionInvokeFunction6(TypedDict):
|
|
3142
|
+
inline_prompt: PromptData
|
|
3143
|
+
function_type: NotRequired[Optional[FunctionTypeEnum]]
|
|
3144
|
+
name: NotRequired[Optional[str]]
|
|
2645
3145
|
"""
|
|
2646
|
-
|
|
3146
|
+
The name of the inline prompt
|
|
2647
3147
|
"""
|
|
2648
3148
|
|
|
2649
3149
|
|
|
3150
|
+
class InvokeFunctionInvokeFunction13(InvokeFunctionInvokeFunction5, InvokeFunctionInvokeFunction7):
|
|
3151
|
+
pass
|
|
3152
|
+
|
|
3153
|
+
|
|
3154
|
+
class InvokeFunctionInvokeFunction14(InvokeFunctionInvokeFunction6, InvokeFunctionInvokeFunction7):
|
|
3155
|
+
pass
|
|
3156
|
+
|
|
3157
|
+
|
|
3158
|
+
InvokeFunction: TypeAlias = Union[
|
|
3159
|
+
InvokeFunctionInvokeFunction8,
|
|
3160
|
+
InvokeFunctionInvokeFunction9,
|
|
3161
|
+
InvokeFunctionInvokeFunction10,
|
|
3162
|
+
InvokeFunctionInvokeFunction11,
|
|
3163
|
+
InvokeFunctionInvokeFunction12,
|
|
3164
|
+
InvokeFunctionInvokeFunction13,
|
|
3165
|
+
InvokeFunctionInvokeFunction14,
|
|
3166
|
+
]
|
|
3167
|
+
"""
|
|
3168
|
+
Options for identifying a function
|
|
3169
|
+
"""
|
|
3170
|
+
|
|
3171
|
+
|
|
2650
3172
|
class Prompt(TypedDict):
|
|
2651
3173
|
id: str
|
|
2652
3174
|
"""
|
|
@@ -2705,7 +3227,7 @@ class RunEval(TypedDict):
|
|
|
2705
3227
|
"""
|
|
2706
3228
|
The dataset to use
|
|
2707
3229
|
"""
|
|
2708
|
-
task:
|
|
3230
|
+
task: Task
|
|
2709
3231
|
scores: Sequence[FunctionId]
|
|
2710
3232
|
"""
|
|
2711
3233
|
The functions to score the eval on
|
|
@@ -2718,7 +3240,7 @@ class RunEval(TypedDict):
|
|
|
2718
3240
|
"""
|
|
2719
3241
|
Optional experiment-level metadata to store about the evaluation. You can later use this to slice & dice across experiments.
|
|
2720
3242
|
"""
|
|
2721
|
-
parent: NotRequired[Optional[
|
|
3243
|
+
parent: NotRequired[Optional[Parent]]
|
|
2722
3244
|
stream: NotRequired[Optional[bool]]
|
|
2723
3245
|
"""
|
|
2724
3246
|
Whether to stream the results of the eval. If true, the request will return two events: one to indicate the experiment has started, and another upon completion. If false, the request will return the evaluation's summary upon completion.
|
|
@@ -2765,14 +3287,16 @@ class RunEval(TypedDict):
|
|
|
2765
3287
|
"""
|
|
2766
3288
|
Optional tags that will be added to the experiment.
|
|
2767
3289
|
"""
|
|
3290
|
+
mcp_auth: NotRequired[Optional[Mapping[str, RunEvalMcpAuth]]]
|
|
2768
3291
|
|
|
2769
3292
|
|
|
2770
|
-
FunctionData = Union[
|
|
3293
|
+
FunctionData: TypeAlias = Union[
|
|
2771
3294
|
FunctionDataFunctionData,
|
|
2772
3295
|
FunctionDataFunctionData1,
|
|
2773
3296
|
GraphData,
|
|
2774
3297
|
FunctionDataFunctionData2,
|
|
2775
3298
|
FunctionDataFunctionData3,
|
|
3299
|
+
FacetData,
|
|
2776
3300
|
]
|
|
2777
3301
|
|
|
2778
3302
|
|