braintrust 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- braintrust/_generated_types.py +139 -10
- braintrust/generated_types.py +5 -1
- braintrust/oai.py +13 -4
- braintrust/span_types.py +1 -0
- braintrust/version.py +2 -2
- braintrust/wrappers/openai.py +54 -3
- braintrust/wrappers/test_openai.py +101 -1
- {braintrust-0.5.3.dist-info → braintrust-0.5.5.dist-info}/METADATA +1 -1
- {braintrust-0.5.3.dist-info → braintrust-0.5.5.dist-info}/RECORD +12 -12
- {braintrust-0.5.3.dist-info → braintrust-0.5.5.dist-info}/WHEEL +0 -0
- {braintrust-0.5.3.dist-info → braintrust-0.5.5.dist-info}/entry_points.txt +0 -0
- {braintrust-0.5.3.dist-info → braintrust-0.5.5.dist-info}/top_level.txt +0 -0
braintrust/_generated_types.py
CHANGED
|
@@ -194,6 +194,10 @@ class BatchedFacetDataFacet(TypedDict):
|
|
|
194
194
|
"""
|
|
195
195
|
The model to use for facet extraction
|
|
196
196
|
"""
|
|
197
|
+
embedding_model: NotRequired[str | None]
|
|
198
|
+
"""
|
|
199
|
+
The embedding model to use for vectorizing facet results.
|
|
200
|
+
"""
|
|
197
201
|
no_match_pattern: NotRequired[str | None]
|
|
198
202
|
"""
|
|
199
203
|
Regex pattern to identify outputs that do not match the facet. If the output matches, the facet will be saved as 'no_match'
|
|
@@ -740,9 +744,32 @@ class FunctionDataFunctionData2(TypedDict):
|
|
|
740
744
|
endpoint: str
|
|
741
745
|
eval_name: str
|
|
742
746
|
parameters: Mapping[str, Any]
|
|
747
|
+
parameters_version: NotRequired[str | None]
|
|
748
|
+
"""
|
|
749
|
+
The version (transaction ID) of the parameters being used
|
|
750
|
+
"""
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
class FunctionDataFunctionData4Schema(TypedDict):
|
|
754
|
+
type: Literal['object']
|
|
755
|
+
properties: Mapping[str, Mapping[str, Any]]
|
|
756
|
+
required: NotRequired[Sequence[str] | None]
|
|
757
|
+
additionalProperties: NotRequired[bool | None]
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
class FunctionDataFunctionData4(TypedDict):
|
|
761
|
+
type: Literal['parameters']
|
|
762
|
+
data: Mapping[str, Any]
|
|
763
|
+
"""
|
|
764
|
+
The parameters data
|
|
765
|
+
"""
|
|
766
|
+
__schema: FunctionDataFunctionData4Schema
|
|
767
|
+
"""
|
|
768
|
+
JSON Schema format for parameters
|
|
769
|
+
"""
|
|
743
770
|
|
|
744
771
|
|
|
745
|
-
FunctionFormat: TypeAlias = Literal['llm', 'code', 'global', 'graph']
|
|
772
|
+
FunctionFormat: TypeAlias = Literal['llm', 'code', 'global', 'graph', 'topic_map']
|
|
746
773
|
|
|
747
774
|
|
|
748
775
|
class FunctionIdFunctionId(TypedDict):
|
|
@@ -807,7 +834,7 @@ FunctionIdRef: TypeAlias = Mapping[str, Any]
|
|
|
807
834
|
|
|
808
835
|
|
|
809
836
|
FunctionObjectType: TypeAlias = Literal[
|
|
810
|
-
'prompt', 'tool', 'scorer', 'task', 'workflow', 'custom_view', 'preprocessor', 'facet', 'classifier'
|
|
837
|
+
'prompt', 'tool', 'scorer', 'task', 'workflow', 'custom_view', 'preprocessor', 'facet', 'classifier', 'parameters'
|
|
811
838
|
]
|
|
812
839
|
|
|
813
840
|
|
|
@@ -815,7 +842,7 @@ FunctionOutputType: TypeAlias = Literal['completion', 'score', 'facet', 'classif
|
|
|
815
842
|
|
|
816
843
|
|
|
817
844
|
FunctionTypeEnum: TypeAlias = Literal[
|
|
818
|
-
'llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet', 'classifier', 'tag'
|
|
845
|
+
'llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet', 'classifier', 'tag', 'parameters'
|
|
819
846
|
]
|
|
820
847
|
"""
|
|
821
848
|
The type of global function. Defaults to 'scorer'.
|
|
@@ -823,7 +850,7 @@ The type of global function. Defaults to 'scorer'.
|
|
|
823
850
|
|
|
824
851
|
|
|
825
852
|
FunctionTypeEnumNullish: TypeAlias = Literal[
|
|
826
|
-
'llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet', 'classifier', 'tag'
|
|
853
|
+
'llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet', 'classifier', 'tag', 'parameters'
|
|
827
854
|
]
|
|
828
855
|
|
|
829
856
|
|
|
@@ -2236,6 +2263,64 @@ class ToolFunctionDefinition(TypedDict):
|
|
|
2236
2263
|
function: ToolFunctionDefinitionFunction
|
|
2237
2264
|
|
|
2238
2265
|
|
|
2266
|
+
class TopicMapReportSettings(TypedDict):
|
|
2267
|
+
algorithm: Literal['hdbscan', 'kmeans', 'hierarchical']
|
|
2268
|
+
dimension_reduction: Literal['umap', 'pca', 'none']
|
|
2269
|
+
vector_field: str
|
|
2270
|
+
embedding_model: str
|
|
2271
|
+
n_clusters: NotRequired[int | None]
|
|
2272
|
+
umap_dimensions: NotRequired[int | None]
|
|
2273
|
+
min_cluster_size: NotRequired[int | None]
|
|
2274
|
+
min_samples: NotRequired[int | None]
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
class TopicMapReportQuerySettings(TypedDict):
|
|
2278
|
+
hierarchy_threshold: NotRequired[int | None]
|
|
2279
|
+
auto_naming: NotRequired[bool | None]
|
|
2280
|
+
skip_cache: NotRequired[bool | None]
|
|
2281
|
+
viz_mode: NotRequired[Literal['bar', 'scatter'] | None]
|
|
2282
|
+
naming_model: NotRequired[str | None]
|
|
2283
|
+
|
|
2284
|
+
|
|
2285
|
+
class TopicMapReportClusterSample(TypedDict):
|
|
2286
|
+
id: str
|
|
2287
|
+
text: str
|
|
2288
|
+
root_span_id: str
|
|
2289
|
+
span_id: str
|
|
2290
|
+
|
|
2291
|
+
|
|
2292
|
+
class TopicMapReportCluster(TypedDict):
|
|
2293
|
+
cluster_id: float
|
|
2294
|
+
parent_cluster_id: NotRequired[float | None]
|
|
2295
|
+
topic_id: str
|
|
2296
|
+
count: float
|
|
2297
|
+
sample_texts: Sequence[str]
|
|
2298
|
+
samples: Sequence[TopicMapReportClusterSample]
|
|
2299
|
+
name: NotRequired[str | None]
|
|
2300
|
+
description: NotRequired[str | None]
|
|
2301
|
+
keywords: NotRequired[Sequence[str] | None]
|
|
2302
|
+
centroid: NotRequired[Sequence[float] | None]
|
|
2303
|
+
parent_id: NotRequired[float | None]
|
|
2304
|
+
is_leaf: NotRequired[bool | None]
|
|
2305
|
+
depth: NotRequired[float | None]
|
|
2306
|
+
|
|
2307
|
+
|
|
2308
|
+
class TopicMapReportEmbeddingPoint(TypedDict):
|
|
2309
|
+
x: float
|
|
2310
|
+
y: float
|
|
2311
|
+
cluster: float
|
|
2312
|
+
text: NotRequired[str | None]
|
|
2313
|
+
|
|
2314
|
+
|
|
2315
|
+
class TopicMapReport(TypedDict):
|
|
2316
|
+
version: Literal[1]
|
|
2317
|
+
created_at: NotRequired[str | None]
|
|
2318
|
+
settings: TopicMapReportSettings
|
|
2319
|
+
query_settings: TopicMapReportQuerySettings
|
|
2320
|
+
clusters: Sequence[TopicMapReportCluster]
|
|
2321
|
+
embedding_points: NotRequired[Sequence[TopicMapReportEmbeddingPoint] | None]
|
|
2322
|
+
|
|
2323
|
+
|
|
2239
2324
|
class TraceScope(TypedDict):
|
|
2240
2325
|
type: Literal['trace']
|
|
2241
2326
|
idle_seconds: NotRequired[float | None]
|
|
@@ -2500,12 +2585,6 @@ class PreprocessorPreprocessor4(PreprocessorPreprocessor1, PreprocessorPreproces
|
|
|
2500
2585
|
Preprocessor: TypeAlias = PreprocessorPreprocessor3 | PreprocessorPreprocessor4
|
|
2501
2586
|
|
|
2502
2587
|
|
|
2503
|
-
class BatchedFacetData(TypedDict):
|
|
2504
|
-
type: Literal['batched_facet']
|
|
2505
|
-
preprocessor: NotRequired[Preprocessor | None]
|
|
2506
|
-
facets: Sequence[BatchedFacetDataFacet]
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
2588
|
ChatCompletionContentPart: TypeAlias = (
|
|
2510
2589
|
ChatCompletionContentPartTextWithTitle
|
|
2511
2590
|
| ChatCompletionContentPartImageWithTitle
|
|
@@ -2761,6 +2840,10 @@ class FacetData(TypedDict):
|
|
|
2761
2840
|
"""
|
|
2762
2841
|
The model to use for facet extraction
|
|
2763
2842
|
"""
|
|
2843
|
+
embedding_model: NotRequired[str | None]
|
|
2844
|
+
"""
|
|
2845
|
+
The embedding model to use for vectorizing facet results.
|
|
2846
|
+
"""
|
|
2764
2847
|
no_match_pattern: NotRequired[str | None]
|
|
2765
2848
|
"""
|
|
2766
2849
|
Regex pattern to identify outputs that do not match the facet. If the output matches, the facet will be saved as 'no_match'
|
|
@@ -3044,11 +3127,54 @@ class SpanAttributes(TypedDict):
|
|
|
3044
3127
|
type: NotRequired[SpanType | None]
|
|
3045
3128
|
|
|
3046
3129
|
|
|
3130
|
+
class TopicMapData(TypedDict):
|
|
3131
|
+
type: Literal['topic_map']
|
|
3132
|
+
source_facet: str
|
|
3133
|
+
"""
|
|
3134
|
+
The facet field name to use as input for classification
|
|
3135
|
+
"""
|
|
3136
|
+
embedding_model: str
|
|
3137
|
+
"""
|
|
3138
|
+
The embedding model to use for embedding facet values
|
|
3139
|
+
"""
|
|
3140
|
+
bundle_key: str
|
|
3141
|
+
"""
|
|
3142
|
+
Key of the topic map bundle in code_bundles bucket
|
|
3143
|
+
"""
|
|
3144
|
+
distance_threshold: NotRequired[float | None]
|
|
3145
|
+
"""
|
|
3146
|
+
Maximum distance to nearest centroid. If exceeded, returns no_match.
|
|
3147
|
+
"""
|
|
3148
|
+
report: NotRequired[TopicMapReport | None]
|
|
3149
|
+
|
|
3150
|
+
|
|
3047
3151
|
class ViewData(TypedDict):
|
|
3048
3152
|
search: NotRequired[ViewDataSearch | None]
|
|
3049
3153
|
custom_charts: NotRequired[Any | None]
|
|
3050
3154
|
|
|
3051
3155
|
|
|
3156
|
+
class BatchedFacetDataTopicMaps(TypedDict):
|
|
3157
|
+
function_name: str
|
|
3158
|
+
"""
|
|
3159
|
+
The name of the topic map function
|
|
3160
|
+
"""
|
|
3161
|
+
topic_map_id: NotRequired[str | None]
|
|
3162
|
+
"""
|
|
3163
|
+
The id of the topic map function
|
|
3164
|
+
"""
|
|
3165
|
+
topic_map_data: TopicMapData
|
|
3166
|
+
|
|
3167
|
+
|
|
3168
|
+
class BatchedFacetData(TypedDict):
|
|
3169
|
+
type: Literal['batched_facet']
|
|
3170
|
+
preprocessor: NotRequired[Preprocessor | None]
|
|
3171
|
+
facets: Sequence[BatchedFacetDataFacet]
|
|
3172
|
+
topic_maps: NotRequired[Mapping[str, BatchedFacetDataTopicMaps] | None]
|
|
3173
|
+
"""
|
|
3174
|
+
Topic maps that depend on facets in this batch, keyed by source facet name
|
|
3175
|
+
"""
|
|
3176
|
+
|
|
3177
|
+
|
|
3052
3178
|
class ExperimentEvent(TypedDict):
|
|
3053
3179
|
id: str
|
|
3054
3180
|
"""
|
|
@@ -3376,6 +3502,7 @@ class View(TypedDict):
|
|
|
3376
3502
|
'datasets',
|
|
3377
3503
|
'dataset',
|
|
3378
3504
|
'prompts',
|
|
3505
|
+
'parameters',
|
|
3379
3506
|
'tools',
|
|
3380
3507
|
'scorers',
|
|
3381
3508
|
'classifiers',
|
|
@@ -3617,6 +3744,8 @@ FunctionData: TypeAlias = (
|
|
|
3617
3744
|
| FunctionDataFunctionData3
|
|
3618
3745
|
| FacetData
|
|
3619
3746
|
| BatchedFacetData
|
|
3747
|
+
| FunctionDataFunctionData4
|
|
3748
|
+
| TopicMapData
|
|
3620
3749
|
)
|
|
3621
3750
|
|
|
3622
3751
|
|
braintrust/generated_types.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Auto-generated file (internal git SHA
|
|
1
|
+
"""Auto-generated file (internal git SHA 2822255bed426d5442adc880a8f71b8a378de3d4) -- do not modify"""
|
|
2
2
|
|
|
3
3
|
from ._generated_types import (
|
|
4
4
|
Acl,
|
|
@@ -101,6 +101,8 @@ from ._generated_types import (
|
|
|
101
101
|
SSEProgressEventData,
|
|
102
102
|
StreamingMode,
|
|
103
103
|
ToolFunctionDefinition,
|
|
104
|
+
TopicMapData,
|
|
105
|
+
TopicMapReport,
|
|
104
106
|
TraceScope,
|
|
105
107
|
TriggeredFunctionState,
|
|
106
108
|
UploadStatus,
|
|
@@ -212,6 +214,8 @@ __all__ = [
|
|
|
212
214
|
"SpanType",
|
|
213
215
|
"StreamingMode",
|
|
214
216
|
"ToolFunctionDefinition",
|
|
217
|
+
"TopicMapData",
|
|
218
|
+
"TopicMapReport",
|
|
215
219
|
"TraceScope",
|
|
216
220
|
"TriggeredFunctionState",
|
|
217
221
|
"UploadStatus",
|
braintrust/oai.py
CHANGED
|
@@ -312,16 +312,25 @@ class ChatCompletionWrapper:
|
|
|
312
312
|
|
|
313
313
|
# pylint: disable=unsubscriptable-object
|
|
314
314
|
if not tool_calls or (tool_delta.get("id") and tool_calls[-1]["id"] != tool_delta.get("id")):
|
|
315
|
+
function_arg = tool_delta.get("function", {})
|
|
315
316
|
tool_calls = (tool_calls or []) + [
|
|
316
317
|
{
|
|
317
|
-
"id":
|
|
318
|
-
"type":
|
|
319
|
-
"function":
|
|
318
|
+
"id": tool_delta.get("id"),
|
|
319
|
+
"type": tool_delta.get("type"),
|
|
320
|
+
"function": {
|
|
321
|
+
"name": function_arg.get("name"),
|
|
322
|
+
"arguments": function_arg.get("arguments") or "",
|
|
323
|
+
},
|
|
320
324
|
}
|
|
321
325
|
]
|
|
322
326
|
else:
|
|
323
327
|
# pylint: disable=unsubscriptable-object
|
|
324
|
-
|
|
328
|
+
# append to existing tool call
|
|
329
|
+
function_arg = tool_delta.get("function", {})
|
|
330
|
+
args = function_arg.get("arguments") or ""
|
|
331
|
+
if isinstance(args, str):
|
|
332
|
+
# pylint: disable=unsubscriptable-object
|
|
333
|
+
tool_calls[-1]["function"]["arguments"] += args
|
|
325
334
|
|
|
326
335
|
return {
|
|
327
336
|
"metrics": metrics,
|
braintrust/span_types.py
CHANGED
braintrust/version.py
CHANGED
braintrust/wrappers/openai.py
CHANGED
|
@@ -11,11 +11,11 @@ from braintrust.logger import NOOP_SPAN
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def _span_type(span: tracing.Span[Any]) -> braintrust.SpanTypeAttribute:
|
|
14
|
-
if span.span_data.type in ["agent", "handoff", "custom"]:
|
|
14
|
+
if span.span_data.type in ["agent", "handoff", "custom", "speech_group"]:
|
|
15
15
|
return braintrust.SpanTypeAttribute.TASK
|
|
16
|
-
elif span.span_data.type in ["function", "guardrail"]:
|
|
16
|
+
elif span.span_data.type in ["function", "guardrail", "mcp_tools"]:
|
|
17
17
|
return braintrust.SpanTypeAttribute.TOOL
|
|
18
|
-
elif span.span_data.type in ["generation", "response"]:
|
|
18
|
+
elif span.span_data.type in ["generation", "response", "transcription", "speech"]:
|
|
19
19
|
return braintrust.SpanTypeAttribute.LLM
|
|
20
20
|
else:
|
|
21
21
|
return braintrust.SpanTypeAttribute.TASK
|
|
@@ -36,6 +36,16 @@ def _span_name(span: tracing.Span[Any]) -> str:
|
|
|
36
36
|
return "Response"
|
|
37
37
|
elif isinstance(span.span_data, tracing.HandoffSpanData):
|
|
38
38
|
return "Handoff"
|
|
39
|
+
elif isinstance(span.span_data, tracing.MCPListToolsSpanData):
|
|
40
|
+
if span.span_data.server:
|
|
41
|
+
return f"List Tools ({span.span_data.server})"
|
|
42
|
+
return "MCP List Tools"
|
|
43
|
+
elif isinstance(span.span_data, tracing.TranscriptionSpanData):
|
|
44
|
+
return "Transcription"
|
|
45
|
+
elif isinstance(span.span_data, tracing.SpeechSpanData):
|
|
46
|
+
return "Speech"
|
|
47
|
+
elif isinstance(span.span_data, tracing.SpeechGroupSpanData):
|
|
48
|
+
return "Speech Group"
|
|
39
49
|
else:
|
|
40
50
|
return "Unknown"
|
|
41
51
|
|
|
@@ -202,6 +212,39 @@ class BraintrustTracingProcessor(tracing.TracingProcessor):
|
|
|
202
212
|
def _custom_log_data(self, span: tracing.Span[tracing.CustomSpanData]) -> dict[str, Any]:
|
|
203
213
|
return span.span_data.data
|
|
204
214
|
|
|
215
|
+
def _mcp_list_tools_log_data(self, span: tracing.Span[tracing.MCPListToolsSpanData]) -> dict[str, Any]:
|
|
216
|
+
return {
|
|
217
|
+
"output": span.span_data.result,
|
|
218
|
+
"metadata": {
|
|
219
|
+
"server": span.span_data.server,
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
def _transcription_log_data(self, span: tracing.Span[tracing.TranscriptionSpanData]) -> dict[str, Any]:
|
|
224
|
+
return {
|
|
225
|
+
"input": span.span_data.input,
|
|
226
|
+
"output": span.span_data.output,
|
|
227
|
+
"metadata": {
|
|
228
|
+
"model": span.span_data.model,
|
|
229
|
+
"model_config": span.span_data.model_config,
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
def _speech_log_data(self, span: tracing.Span[tracing.SpeechSpanData]) -> dict[str, Any]:
|
|
234
|
+
return {
|
|
235
|
+
"input": span.span_data.input,
|
|
236
|
+
"output": span.span_data.output,
|
|
237
|
+
"metadata": {
|
|
238
|
+
"model": span.span_data.model,
|
|
239
|
+
"model_config": span.span_data.model_config,
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
def _speech_group_log_data(self, span: tracing.Span[tracing.SpeechGroupSpanData]) -> dict[str, Any]:
|
|
244
|
+
return {
|
|
245
|
+
"input": span.span_data.input,
|
|
246
|
+
}
|
|
247
|
+
|
|
205
248
|
def _log_data(self, span: tracing.Span[Any]) -> dict[str, Any]:
|
|
206
249
|
if isinstance(span.span_data, tracing.AgentSpanData):
|
|
207
250
|
return self._agent_log_data(span)
|
|
@@ -217,6 +260,14 @@ class BraintrustTracingProcessor(tracing.TracingProcessor):
|
|
|
217
260
|
return self._generation_log_data(span)
|
|
218
261
|
elif isinstance(span.span_data, tracing.CustomSpanData):
|
|
219
262
|
return self._custom_log_data(span)
|
|
263
|
+
elif isinstance(span.span_data, tracing.MCPListToolsSpanData):
|
|
264
|
+
return self._mcp_list_tools_log_data(span)
|
|
265
|
+
elif isinstance(span.span_data, tracing.TranscriptionSpanData):
|
|
266
|
+
return self._transcription_log_data(span)
|
|
267
|
+
elif isinstance(span.span_data, tracing.SpeechSpanData):
|
|
268
|
+
return self._speech_log_data(span)
|
|
269
|
+
elif isinstance(span.span_data, tracing.SpeechGroupSpanData):
|
|
270
|
+
return self._speech_group_log_data(span)
|
|
220
271
|
else:
|
|
221
272
|
return {}
|
|
222
273
|
|
|
@@ -5,6 +5,7 @@ import braintrust
|
|
|
5
5
|
import openai
|
|
6
6
|
import pytest
|
|
7
7
|
from braintrust import logger, wrap_openai
|
|
8
|
+
from braintrust.oai import ChatCompletionWrapper
|
|
8
9
|
from braintrust.test_helpers import assert_dict_matches, init_test_logger
|
|
9
10
|
from braintrust.wrappers.test_utils import assert_metrics_are_valid, run_in_subprocess, verify_autoinstrument_script
|
|
10
11
|
from openai import AsyncOpenAI
|
|
@@ -377,7 +378,6 @@ def test_openai_responses_sparse_indices(memory_logger):
|
|
|
377
378
|
# No spans should be generated from this unit test
|
|
378
379
|
assert not memory_logger.pop()
|
|
379
380
|
|
|
380
|
-
|
|
381
381
|
@pytest.mark.vcr
|
|
382
382
|
def test_openai_embeddings(memory_logger):
|
|
383
383
|
assert not memory_logger.pop()
|
|
@@ -1537,6 +1537,7 @@ async def test_braintrust_tracing_processor_concurrency_bug(memory_logger):
|
|
|
1537
1537
|
|
|
1538
1538
|
@pytest.mark.asyncio
|
|
1539
1539
|
@pytest.mark.vcr
|
|
1540
|
+
@pytest.mark.skip(reason="OAI Implementation changed, skipping until update")
|
|
1540
1541
|
async def test_agents_tool_openai_nested_spans(memory_logger):
|
|
1541
1542
|
"""Test that OpenAI calls inside agent tools are properly nested under the tool span."""
|
|
1542
1543
|
pytest.importorskip("agents", reason="agents package not available")
|
|
@@ -1933,3 +1934,102 @@ class TestAutoInstrumentOpenAI:
|
|
|
1933
1934
|
def test_auto_instrument_openai(self):
|
|
1934
1935
|
"""Test auto_instrument patches OpenAI, creates spans, and uninstrument works."""
|
|
1935
1936
|
verify_autoinstrument_script("test_auto_openai.py")
|
|
1937
|
+
|
|
1938
|
+
class TestZAICompatibleOpenAI:
|
|
1939
|
+
"""Tests for validating some ZAI compatibility with OpenAI wrapper."""
|
|
1940
|
+
|
|
1941
|
+
def test_chat_completion_streaming_none_arguments(self, memory_logger):
|
|
1942
|
+
"""Test that ChatCompletionWrapper handles None arguments in tool calls (e.g., GLM-4.6 behavior)."""
|
|
1943
|
+
assert not memory_logger.pop()
|
|
1944
|
+
|
|
1945
|
+
# Simulate streaming results with None arguments in tool calls
|
|
1946
|
+
# This mimics the behavior of GLM-4.6 which returns {'arguments': None, 'name': 'weather'}
|
|
1947
|
+
all_results = [
|
|
1948
|
+
# First chunk: initial tool call with None arguments
|
|
1949
|
+
{
|
|
1950
|
+
"choices": [
|
|
1951
|
+
{
|
|
1952
|
+
"delta": {
|
|
1953
|
+
"role": "assistant",
|
|
1954
|
+
"tool_calls": [
|
|
1955
|
+
{
|
|
1956
|
+
"id": "call_123",
|
|
1957
|
+
"type": "function",
|
|
1958
|
+
"function": {
|
|
1959
|
+
"name": "get_weather",
|
|
1960
|
+
"arguments": None, # GLM-4.6 returns None here
|
|
1961
|
+
},
|
|
1962
|
+
}
|
|
1963
|
+
],
|
|
1964
|
+
},
|
|
1965
|
+
"finish_reason": None,
|
|
1966
|
+
}
|
|
1967
|
+
],
|
|
1968
|
+
},
|
|
1969
|
+
# Second chunk: subsequent tool call arguments (also None)
|
|
1970
|
+
{
|
|
1971
|
+
"choices": [
|
|
1972
|
+
{
|
|
1973
|
+
"delta": {
|
|
1974
|
+
"tool_calls": [
|
|
1975
|
+
{
|
|
1976
|
+
"function": {
|
|
1977
|
+
"arguments": None, # Subsequent chunks can also have None
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
],
|
|
1981
|
+
},
|
|
1982
|
+
"finish_reason": None,
|
|
1983
|
+
}
|
|
1984
|
+
],
|
|
1985
|
+
},
|
|
1986
|
+
# Third chunk: actual arguments
|
|
1987
|
+
{
|
|
1988
|
+
"choices": [
|
|
1989
|
+
{
|
|
1990
|
+
"delta": {
|
|
1991
|
+
"tool_calls": [
|
|
1992
|
+
{
|
|
1993
|
+
"function": {
|
|
1994
|
+
"arguments": '{"city": "New York"}',
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
],
|
|
1998
|
+
},
|
|
1999
|
+
"finish_reason": None,
|
|
2000
|
+
}
|
|
2001
|
+
],
|
|
2002
|
+
},
|
|
2003
|
+
# Final chunk
|
|
2004
|
+
{
|
|
2005
|
+
"choices": [
|
|
2006
|
+
{
|
|
2007
|
+
"delta": {},
|
|
2008
|
+
"finish_reason": "tool_calls",
|
|
2009
|
+
}
|
|
2010
|
+
],
|
|
2011
|
+
},
|
|
2012
|
+
]
|
|
2013
|
+
|
|
2014
|
+
# Process the results
|
|
2015
|
+
wrapper = ChatCompletionWrapper(None, None)
|
|
2016
|
+
result = wrapper._postprocess_streaming_results(all_results)
|
|
2017
|
+
|
|
2018
|
+
# Verify the output was built correctly
|
|
2019
|
+
assert "output" in result
|
|
2020
|
+
assert len(result["output"]) == 1
|
|
2021
|
+
message = result["output"][0]["message"]
|
|
2022
|
+
assert message["role"] == "assistant"
|
|
2023
|
+
assert message["tool_calls"] is not None
|
|
2024
|
+
assert len(message["tool_calls"]) == 1
|
|
2025
|
+
|
|
2026
|
+
# Verify the tool call was assembled correctly despite None arguments
|
|
2027
|
+
tool_call = message["tool_calls"][0]
|
|
2028
|
+
assert tool_call["id"] == "call_123"
|
|
2029
|
+
assert tool_call["type"] == "function"
|
|
2030
|
+
assert tool_call["function"]["name"] == "get_weather"
|
|
2031
|
+
# The arguments should be the concatenation: "" + "" + '{"city": "New York"}'
|
|
2032
|
+
assert tool_call["function"]["arguments"] == '{"city": "New York"}'
|
|
2033
|
+
|
|
2034
|
+
# No spans should be generated from this unit test
|
|
2035
|
+
assert not memory_logger.pop()
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
braintrust/__init__.py,sha256=DU4gzkV0R3nsWxp2da7iQS1MM_T9xHkrKSQE6nAnZbE,2627
|
|
2
|
-
braintrust/_generated_types.py,sha256=
|
|
2
|
+
braintrust/_generated_types.py,sha256=d1z6xRHPc0iqifiaoCh7YwDQNSxKZAhM2KYmG0Fyrvk,104819
|
|
3
3
|
braintrust/audit.py,sha256=3GQKzuTcFquYdrJtABM-k3xMlOIqgVkfG6UyeQ8_028,461
|
|
4
4
|
braintrust/auto.py,sha256=wf4Jb7hYoGS0Agpx-YjUYEW7wwwUpyAJDp-l3A-y6c0,4792
|
|
5
5
|
braintrust/aws.py,sha256=OBz_SRyopgpCDSNvETLypzGwTXk-bNLn-Eisevnjfwo,377
|
|
@@ -9,14 +9,14 @@ braintrust/context.py,sha256=bOo1Li29lUsi2DOOllIDar0oRuQNbkLNzJ7cYq5JTbo,4126
|
|
|
9
9
|
braintrust/db_fields.py,sha256=AX-K5t7KqO-xHHOfVRv8bn1ww7gZd3RNIFfANkZ2W0U,709
|
|
10
10
|
braintrust/framework.py,sha256=KoXFKCprfeEHq-AqjarENcmHHcTuvr4PScWtMbIQ6zg,62635
|
|
11
11
|
braintrust/framework2.py,sha256=o0igz4vXbmn0jHJPhDYvx14rFnI3ntV8H6VJfyJYRtM,16542
|
|
12
|
-
braintrust/generated_types.py,sha256=
|
|
12
|
+
braintrust/generated_types.py,sha256=WExtraVBQZMS5DcCVSQl_k66p6VUY7cK7u1SSQhc_to,5127
|
|
13
13
|
braintrust/git_fields.py,sha256=au5ayyuvt7y_ojE9LC98ypTZd3RgFdjhRc8eFxcjnto,1434
|
|
14
14
|
braintrust/gitutil.py,sha256=RsW7cawJMgaAaTw6WeB1sShyfflkPb7yH2x7yuRv10c,5642
|
|
15
15
|
braintrust/http_headers.py,sha256=9ZsDcsAKG04SGowsgchZktD6rG_oSTKWa8QyGUPA4xE,154
|
|
16
16
|
braintrust/id_gen.py,sha256=4UWLWRhksf76IkYi4cKACSaQ3yNgausrMRlhiurhy74,1590
|
|
17
17
|
braintrust/logger.py,sha256=ofx0_8ywVk_iyyqOH-6kCgRaYmm_sO7guVJ5hW2ISNg,217467
|
|
18
18
|
braintrust/merge_row_batch.py,sha256=mCutumLDOpH8ArP_4K5swP93mve7gmgfAnQdkdjUZ5E,6271
|
|
19
|
-
braintrust/oai.py,sha256=
|
|
19
|
+
braintrust/oai.py,sha256=ispeHAeykwF3VmGMnaVKZRn7Q80DMtpM7nitFrtybRQ,38193
|
|
20
20
|
braintrust/object.py,sha256=vYLyYWncsqLD00zffZUJwGTSkcJF9IIXmgIzrx3Np5c,632
|
|
21
21
|
braintrust/parameters.py,sha256=sQWfw18QXdPSnMHsF7aRrPmP7Zx6HEz9vaTUXWreudg,5911
|
|
22
22
|
braintrust/prompt.py,sha256=pLzhXoBp7ebxNraZADflR4YQMW5Ycjmt5ucK80P4_h0,1875
|
|
@@ -30,7 +30,7 @@ braintrust/span_identifier_v1.py,sha256=eR-dHda0MurdOlghv7-CLSh7eVpNQigbDSQxPpLC
|
|
|
30
30
|
braintrust/span_identifier_v2.py,sha256=2dLc-Vz8iWLISmL_-ebCyWnY-ysA7sMnBsQtKqzMHYY,8981
|
|
31
31
|
braintrust/span_identifier_v3.py,sha256=RAvOK0lK0huH952kI5X1Q9TaAloD5to8jgTCuYMMw6o,10356
|
|
32
32
|
braintrust/span_identifier_v4.py,sha256=uFT-OdzySo4uCeAaJC3VqH55Oy433xZGBdK2hiEsm2w,10044
|
|
33
|
-
braintrust/span_types.py,sha256=
|
|
33
|
+
braintrust/span_types.py,sha256=TtUnrXTVmB1-9_ybaloCoVH9wIbhH0EDEqKMEA5kTDI,398
|
|
34
34
|
braintrust/test_bt_json.py,sha256=pokqmFSQ3m8aB0XN1EEsS-zbv_N4KCYzad6VyOFtwPw,25188
|
|
35
35
|
braintrust/test_context.py,sha256=4wZOhwzGgAxt-CcN2izAxhgcY975oh_HUMkZXRwwTys,42754
|
|
36
36
|
braintrust/test_framework.py,sha256=rPjJYfVzRSbjf8e0irVfCnE3q5Rl1O_2Dy2dPyC9-lo,18674
|
|
@@ -51,7 +51,7 @@ braintrust/test_util.py,sha256=SuSKTmvNyaR9Rbgf2TYCUWxJpZHoA2BMx6n4nQfV_pM,8221
|
|
|
51
51
|
braintrust/test_version.py,sha256=hk5JKjEFbNJ_ONc1VEkqHquflzre34RpFhCEYLTK8iA,1051
|
|
52
52
|
braintrust/trace.py,sha256=PHxfaHApGP_MPMIndZw7atIa2iwKPETUoG-TbF2dv6A,13767
|
|
53
53
|
braintrust/util.py,sha256=S-qMBNsT36r_3pJ4LNKZ-vvHRlJwy8Wy7M7NAdfNOug,8919
|
|
54
|
-
braintrust/version.py,sha256=
|
|
54
|
+
braintrust/version.py,sha256=YMyThQLGiDPCF_KsLHczZxIgOIFFyy6xDhZycKleyAo,117
|
|
55
55
|
braintrust/xact_ids.py,sha256=bdyp88HjlyIkglgLSqYlCYscdSH6EWVyE14sR90Xl1s,658
|
|
56
56
|
braintrust/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
57
57
|
braintrust/cli/__main__.py,sha256=wCBKHGVmn3IT_yMXk5qfDwyI2SV2gf1tLr0NTxm9T8k,1519
|
|
@@ -100,7 +100,7 @@ braintrust/wrappers/dspy.py,sha256=DGvDkSteOqhNhD5R3XgYcIKAI1-Ui2WYUbjqBl88C2Y,1
|
|
|
100
100
|
braintrust/wrappers/langchain.py,sha256=0aY5LuVA7BPkgWA0N6CwPG9EaPqRmVVfEPaM1kN4XZY,5028
|
|
101
101
|
braintrust/wrappers/langsmith_wrapper.py,sha256=mWhBnY6KypOlSQw4NrCwzif0Oe6SHV6LYGGiwiz-EJQ,17956
|
|
102
102
|
braintrust/wrappers/litellm.py,sha256=SxyDrji84OReBGiIf7aP51iIIF8ocD44QmmnYJwVCbo,25168
|
|
103
|
-
braintrust/wrappers/openai.py,sha256=
|
|
103
|
+
braintrust/wrappers/openai.py,sha256=nrWSK7iyG7Atkl5OeVUljGfHCj625HMgtAfFSD94KKk,12754
|
|
104
104
|
braintrust/wrappers/pydantic_ai.py,sha256=dukJCdcfkRc0ue8nPHpI5DogwNvL9kooGJEbri2W97w,48435
|
|
105
105
|
braintrust/wrappers/test_agno.py,sha256=ad5w6CEWzNwmzbF4NwSvHWhw1SpSjrt-X29pCmBRkhY,3772
|
|
106
106
|
braintrust/wrappers/test_anthropic.py,sha256=VkmlP3_FrZ12x5YWxcbx11A-TEZUghGQeQ4OuT3R4Oc,21409
|
|
@@ -109,7 +109,7 @@ braintrust/wrappers/test_google_genai.py,sha256=BKkPfLFk2JG1UNJ3mFKzv5OjioWqo7EQ
|
|
|
109
109
|
braintrust/wrappers/test_langsmith_wrapper.py,sha256=wEbPNy4o7VVvcuHcsCJ-sy2EATvBxhUXTYFBQNkKCjs,10449
|
|
110
110
|
braintrust/wrappers/test_litellm.py,sha256=MKukVH-C-Mwc0-cVJZhkLdXwiXupXHc8EiWYKq0X8V0,23620
|
|
111
111
|
braintrust/wrappers/test_oai_attachments.py,sha256=_EtNXjQxPgqXmj6UYMZn9GF4GDZf8m_1_TrwiEk7HWQ,11100
|
|
112
|
-
braintrust/wrappers/test_openai.py,sha256=
|
|
112
|
+
braintrust/wrappers/test_openai.py,sha256=wJjsFRZFTFgePrxgPwzUxNNg5ZW6djX25qxGQDX09Us,73433
|
|
113
113
|
braintrust/wrappers/test_openrouter.py,sha256=8HUfILPugOMqcvttpq76KQrynFb0xZpazvta7TTSF6A,3849
|
|
114
114
|
braintrust/wrappers/test_pydantic_ai_integration.py,sha256=xUSkiY5HUI-Z9G_etjyX7rGlOjBEdEY9CwVHyWM3xEE,104460
|
|
115
115
|
braintrust/wrappers/test_pydantic_ai_wrap_openai.py,sha256=OO5NrbothkMr4v2sZ-EZLH7-yLj3k6TfdLG4uzXAsQk,5090
|
|
@@ -125,8 +125,8 @@ braintrust/wrappers/claude_agent_sdk/__init__.py,sha256=4FUE59ii39jVfhMAfkOcU-TJ
|
|
|
125
125
|
braintrust/wrappers/claude_agent_sdk/_wrapper.py,sha256=UNZBcgTu7X71zvJKy6e-QQFuz9j8kRS6Kd1VOh3OOXA,17755
|
|
126
126
|
braintrust/wrappers/claude_agent_sdk/test_wrapper.py,sha256=RmSzTfDC3tkepEbc9S_KYFITsjGVyllYE1fF9roSqBk,10779
|
|
127
127
|
braintrust/wrappers/google_genai/__init__.py,sha256=C7MzKUr17CEcIbP0kg9L9IjN-6suE3NQ-oSYAjXdR_g,15832
|
|
128
|
-
braintrust-0.5.
|
|
129
|
-
braintrust-0.5.
|
|
130
|
-
braintrust-0.5.
|
|
131
|
-
braintrust-0.5.
|
|
132
|
-
braintrust-0.5.
|
|
128
|
+
braintrust-0.5.5.dist-info/METADATA,sha256=-AmNJoVHW9evtVRxgy1OTIJvZHN7zVbfMEugdDEEPVU,3753
|
|
129
|
+
braintrust-0.5.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
130
|
+
braintrust-0.5.5.dist-info/entry_points.txt,sha256=Zpc0_09g5xm8as5jHqqFq7fhwO0xHSNct_TrEMONS7Q,60
|
|
131
|
+
braintrust-0.5.5.dist-info/top_level.txt,sha256=hw1-y-UFMf60RzAr8x_eM7SThbIuWfQsQIbVvqSF83A,11
|
|
132
|
+
braintrust-0.5.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|