braintrust 0.3.15__py3-none-any.whl → 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- braintrust/_generated_types.py +737 -672
- braintrust/audit.py +2 -2
- braintrust/bt_json.py +178 -19
- braintrust/cli/eval.py +6 -7
- braintrust/cli/push.py +11 -11
- braintrust/context.py +12 -17
- braintrust/contrib/temporal/__init__.py +16 -27
- braintrust/contrib/temporal/test_temporal.py +8 -3
- braintrust/devserver/auth.py +8 -8
- braintrust/devserver/cache.py +3 -4
- braintrust/devserver/cors.py +8 -7
- braintrust/devserver/dataset.py +3 -5
- braintrust/devserver/eval_hooks.py +7 -6
- braintrust/devserver/schemas.py +22 -19
- braintrust/devserver/server.py +19 -12
- braintrust/devserver/test_cached_login.py +4 -4
- braintrust/framework.py +139 -142
- braintrust/framework2.py +88 -87
- braintrust/functions/invoke.py +66 -59
- braintrust/functions/stream.py +3 -2
- braintrust/generated_types.py +3 -1
- braintrust/git_fields.py +11 -11
- braintrust/gitutil.py +2 -3
- braintrust/graph_util.py +10 -10
- braintrust/id_gen.py +2 -2
- braintrust/logger.py +373 -471
- braintrust/merge_row_batch.py +10 -9
- braintrust/oai.py +21 -20
- braintrust/otel/__init__.py +49 -49
- braintrust/otel/context.py +16 -30
- braintrust/otel/test_distributed_tracing.py +14 -11
- braintrust/otel/test_otel_bt_integration.py +32 -31
- braintrust/parameters.py +8 -8
- braintrust/prompt.py +14 -14
- braintrust/prompt_cache/disk_cache.py +5 -4
- braintrust/prompt_cache/lru_cache.py +3 -2
- braintrust/prompt_cache/prompt_cache.py +13 -14
- braintrust/queue.py +4 -4
- braintrust/score.py +4 -4
- braintrust/serializable_data_class.py +4 -4
- braintrust/span_identifier_v1.py +1 -2
- braintrust/span_identifier_v2.py +3 -4
- braintrust/span_identifier_v3.py +23 -20
- braintrust/span_identifier_v4.py +34 -25
- braintrust/test_bt_json.py +644 -0
- braintrust/test_framework.py +72 -6
- braintrust/test_helpers.py +5 -5
- braintrust/test_id_gen.py +2 -3
- braintrust/test_logger.py +211 -107
- braintrust/test_otel.py +61 -53
- braintrust/test_queue.py +0 -1
- braintrust/test_score.py +1 -3
- braintrust/test_span_components.py +29 -44
- braintrust/util.py +9 -8
- braintrust/version.py +2 -2
- braintrust/wrappers/_anthropic_utils.py +4 -4
- braintrust/wrappers/agno/__init__.py +3 -4
- braintrust/wrappers/agno/agent.py +1 -2
- braintrust/wrappers/agno/function_call.py +1 -2
- braintrust/wrappers/agno/model.py +1 -2
- braintrust/wrappers/agno/team.py +1 -2
- braintrust/wrappers/agno/utils.py +12 -12
- braintrust/wrappers/anthropic.py +7 -8
- braintrust/wrappers/claude_agent_sdk/__init__.py +3 -4
- braintrust/wrappers/claude_agent_sdk/_wrapper.py +29 -27
- braintrust/wrappers/dspy.py +15 -17
- braintrust/wrappers/google_genai/__init__.py +17 -30
- braintrust/wrappers/langchain.py +22 -24
- braintrust/wrappers/litellm.py +4 -3
- braintrust/wrappers/openai.py +15 -15
- braintrust/wrappers/pydantic_ai.py +225 -110
- braintrust/wrappers/test_agno.py +0 -1
- braintrust/wrappers/test_dspy.py +0 -1
- braintrust/wrappers/test_google_genai.py +64 -4
- braintrust/wrappers/test_litellm.py +0 -1
- braintrust/wrappers/test_pydantic_ai_integration.py +819 -22
- {braintrust-0.3.15.dist-info → braintrust-0.4.1.dist-info}/METADATA +3 -2
- braintrust-0.4.1.dist-info/RECORD +121 -0
- braintrust-0.3.15.dist-info/RECORD +0 -120
- {braintrust-0.3.15.dist-info → braintrust-0.4.1.dist-info}/WHEEL +0 -0
- {braintrust-0.3.15.dist-info → braintrust-0.4.1.dist-info}/entry_points.txt +0 -0
- {braintrust-0.3.15.dist-info → braintrust-0.4.1.dist-info}/top_level.txt +0 -0
braintrust/logger.py
CHANGED
|
@@ -9,7 +9,6 @@ import inspect
|
|
|
9
9
|
import io
|
|
10
10
|
import json
|
|
11
11
|
import logging
|
|
12
|
-
import math
|
|
13
12
|
import os
|
|
14
13
|
import sys
|
|
15
14
|
import textwrap
|
|
@@ -19,24 +18,16 @@ import traceback
|
|
|
19
18
|
import types
|
|
20
19
|
import uuid
|
|
21
20
|
from abc import ABC, abstractmethod
|
|
21
|
+
from collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence
|
|
22
22
|
from functools import partial, wraps
|
|
23
23
|
from multiprocessing import cpu_count
|
|
24
24
|
from types import TracebackType
|
|
25
25
|
from typing import (
|
|
26
26
|
Any,
|
|
27
|
-
Callable,
|
|
28
27
|
Dict,
|
|
29
28
|
Generic,
|
|
30
|
-
Iterator,
|
|
31
|
-
List,
|
|
32
29
|
Literal,
|
|
33
|
-
Mapping,
|
|
34
|
-
MutableMapping,
|
|
35
30
|
Optional,
|
|
36
|
-
Sequence,
|
|
37
|
-
Set,
|
|
38
|
-
Tuple,
|
|
39
|
-
Type,
|
|
40
31
|
TypedDict,
|
|
41
32
|
TypeVar,
|
|
42
33
|
Union,
|
|
@@ -54,7 +45,7 @@ from requests.adapters import HTTPAdapter
|
|
|
54
45
|
from urllib3.util.retry import Retry
|
|
55
46
|
|
|
56
47
|
from . import context, id_gen
|
|
57
|
-
from .bt_json import bt_dumps,
|
|
48
|
+
from .bt_json import bt_dumps, bt_safe_deep_copy
|
|
58
49
|
from .db_fields import (
|
|
59
50
|
ASYNC_SCORING_CONTROL_FIELD,
|
|
60
51
|
AUDIT_METADATA_FIELD,
|
|
@@ -107,7 +98,7 @@ from .util import (
|
|
|
107
98
|
REDACTION_FIELDS = ["input", "output", "expected", "metadata", "context", "scores", "metrics"]
|
|
108
99
|
from .xact_ids import prettify_xact
|
|
109
100
|
|
|
110
|
-
Metadata =
|
|
101
|
+
Metadata = dict[str, Any]
|
|
111
102
|
DATA_API_VERSION = 2
|
|
112
103
|
|
|
113
104
|
T = TypeVar("T")
|
|
@@ -161,12 +152,12 @@ class Span(Exportable, contextlib.AbstractContextManager, ABC):
|
|
|
161
152
|
@abstractmethod
|
|
162
153
|
def start_span(
|
|
163
154
|
self,
|
|
164
|
-
name:
|
|
165
|
-
type:
|
|
166
|
-
span_attributes:
|
|
167
|
-
start_time:
|
|
168
|
-
set_current:
|
|
169
|
-
parent:
|
|
155
|
+
name: str | None = None,
|
|
156
|
+
type: SpanTypeAttribute | None = None,
|
|
157
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
158
|
+
start_time: float | None = None,
|
|
159
|
+
set_current: bool | None = None,
|
|
160
|
+
parent: str | None = None,
|
|
170
161
|
**event: Any,
|
|
171
162
|
) -> "Span":
|
|
172
163
|
"""Create a new span. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row.
|
|
@@ -224,7 +215,7 @@ class Span(Exportable, contextlib.AbstractContextManager, ABC):
|
|
|
224
215
|
"""
|
|
225
216
|
|
|
226
217
|
@abstractmethod
|
|
227
|
-
def end(self, end_time:
|
|
218
|
+
def end(self, end_time: float | None = None) -> float:
|
|
228
219
|
"""Log an end time to the span (defaults to the current time). Returns the logged time.
|
|
229
220
|
|
|
230
221
|
Will be invoked automatically if the span is bound to a context manager.
|
|
@@ -238,15 +229,15 @@ class Span(Exportable, contextlib.AbstractContextManager, ABC):
|
|
|
238
229
|
"""Flush any pending rows to the server."""
|
|
239
230
|
|
|
240
231
|
@abstractmethod
|
|
241
|
-
def close(self, end_time:
|
|
232
|
+
def close(self, end_time: float | None = None) -> float:
|
|
242
233
|
"""Alias for `end`."""
|
|
243
234
|
|
|
244
235
|
@abstractmethod
|
|
245
236
|
def set_attributes(
|
|
246
237
|
self,
|
|
247
|
-
name:
|
|
248
|
-
type:
|
|
249
|
-
span_attributes:
|
|
238
|
+
name: str | None = None,
|
|
239
|
+
type: SpanTypeAttribute | None = None,
|
|
240
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
250
241
|
) -> None:
|
|
251
242
|
"""Set the span's name, type, or other attributes. These attributes will be attached to all log events within the span.
|
|
252
243
|
The attributes are equivalent to the arguments to start_span.
|
|
@@ -279,6 +270,10 @@ class _NoopSpan(Span):
|
|
|
279
270
|
def id(self):
|
|
280
271
|
return ""
|
|
281
272
|
|
|
273
|
+
@property
|
|
274
|
+
def propagated_event(self):
|
|
275
|
+
return None
|
|
276
|
+
|
|
282
277
|
def log(self, **event: Any):
|
|
283
278
|
pass
|
|
284
279
|
|
|
@@ -287,17 +282,17 @@ class _NoopSpan(Span):
|
|
|
287
282
|
|
|
288
283
|
def start_span(
|
|
289
284
|
self,
|
|
290
|
-
name:
|
|
291
|
-
type:
|
|
292
|
-
span_attributes:
|
|
293
|
-
start_time:
|
|
294
|
-
set_current:
|
|
295
|
-
parent:
|
|
285
|
+
name: str | None = None,
|
|
286
|
+
type: SpanTypeAttribute | None = None,
|
|
287
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
288
|
+
start_time: float | None = None,
|
|
289
|
+
set_current: bool | None = None,
|
|
290
|
+
parent: str | None = None,
|
|
296
291
|
**event: Any,
|
|
297
292
|
):
|
|
298
293
|
return self
|
|
299
294
|
|
|
300
|
-
def end(self, end_time:
|
|
295
|
+
def end(self, end_time: float | None = None) -> float:
|
|
301
296
|
return end_time or time.time()
|
|
302
297
|
|
|
303
298
|
def export(self):
|
|
@@ -312,14 +307,14 @@ class _NoopSpan(Span):
|
|
|
312
307
|
def flush(self):
|
|
313
308
|
pass
|
|
314
309
|
|
|
315
|
-
def close(self, end_time:
|
|
310
|
+
def close(self, end_time: float | None = None) -> float:
|
|
316
311
|
return self.end(end_time)
|
|
317
312
|
|
|
318
313
|
def set_attributes(
|
|
319
314
|
self,
|
|
320
|
-
name:
|
|
321
|
-
type:
|
|
322
|
-
span_attributes:
|
|
315
|
+
name: str | None = None,
|
|
316
|
+
type: SpanTypeAttribute | None = None,
|
|
317
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
323
318
|
):
|
|
324
319
|
pass
|
|
325
320
|
|
|
@@ -334,9 +329,9 @@ class _NoopSpan(Span):
|
|
|
334
329
|
|
|
335
330
|
def __exit__(
|
|
336
331
|
self,
|
|
337
|
-
exc_type:
|
|
338
|
-
exc_value:
|
|
339
|
-
traceback:
|
|
332
|
+
exc_type: type[BaseException] | None,
|
|
333
|
+
exc_value: BaseException | None,
|
|
334
|
+
traceback: TracebackType | None,
|
|
340
335
|
):
|
|
341
336
|
pass
|
|
342
337
|
|
|
@@ -348,11 +343,11 @@ NOOP_SPAN_PERMALINK = "https://www.braintrust.dev/noop-span"
|
|
|
348
343
|
class BraintrustState:
|
|
349
344
|
def __init__(self):
|
|
350
345
|
self.id = str(uuid.uuid4())
|
|
351
|
-
self.current_experiment:
|
|
352
|
-
self.current_logger: contextvars.ContextVar[
|
|
346
|
+
self.current_experiment: Experiment | None = None
|
|
347
|
+
self.current_logger: contextvars.ContextVar[Logger | None] = contextvars.ContextVar(
|
|
353
348
|
"braintrust_current_logger", default=None
|
|
354
349
|
)
|
|
355
|
-
self.current_parent: contextvars.ContextVar[
|
|
350
|
+
self.current_parent: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
|
356
351
|
"braintrust_current_parent", default=None
|
|
357
352
|
)
|
|
358
353
|
self.current_span: contextvars.ContextVar[Span] = contextvars.ContextVar(
|
|
@@ -402,20 +397,20 @@ class BraintrustState:
|
|
|
402
397
|
)
|
|
403
398
|
|
|
404
399
|
def reset_login_info(self):
|
|
405
|
-
self.app_url:
|
|
406
|
-
self.app_public_url:
|
|
407
|
-
self.login_token:
|
|
408
|
-
self.org_id:
|
|
409
|
-
self.org_name:
|
|
410
|
-
self.api_url:
|
|
411
|
-
self.proxy_url:
|
|
400
|
+
self.app_url: str | None = None
|
|
401
|
+
self.app_public_url: str | None = None
|
|
402
|
+
self.login_token: str | None = None
|
|
403
|
+
self.org_id: str | None = None
|
|
404
|
+
self.org_name: str | None = None
|
|
405
|
+
self.api_url: str | None = None
|
|
406
|
+
self.proxy_url: str | None = None
|
|
412
407
|
self.logged_in: bool = False
|
|
413
|
-
self.git_metadata_settings:
|
|
408
|
+
self.git_metadata_settings: GitMetadataSettings | None = None
|
|
414
409
|
|
|
415
|
-
self._app_conn:
|
|
416
|
-
self._api_conn:
|
|
417
|
-
self._proxy_conn:
|
|
418
|
-
self._user_info:
|
|
410
|
+
self._app_conn: HTTPConnection | None = None
|
|
411
|
+
self._api_conn: HTTPConnection | None = None
|
|
412
|
+
self._proxy_conn: HTTPConnection | None = None
|
|
413
|
+
self._user_info: Mapping[str, Any] | None = None
|
|
419
414
|
|
|
420
415
|
def reset_parent_state(self):
|
|
421
416
|
# reset possible parent state for tests
|
|
@@ -480,9 +475,9 @@ class BraintrustState:
|
|
|
480
475
|
|
|
481
476
|
def login(
|
|
482
477
|
self,
|
|
483
|
-
app_url:
|
|
484
|
-
api_key:
|
|
485
|
-
org_name:
|
|
478
|
+
app_url: str | None = None,
|
|
479
|
+
api_key: str | None = None,
|
|
480
|
+
org_name: str | None = None,
|
|
486
481
|
force_login: bool = False,
|
|
487
482
|
) -> None:
|
|
488
483
|
if not force_login and self.logged_in:
|
|
@@ -558,7 +553,7 @@ class BraintrustState:
|
|
|
558
553
|
bg_logger = self._global_bg_logger.get()
|
|
559
554
|
bg_logger.enforce_queue_size_limit(enforce)
|
|
560
555
|
|
|
561
|
-
def set_masking_function(self, masking_function:
|
|
556
|
+
def set_masking_function(self, masking_function: Callable[[Any], Any] | None) -> None:
|
|
562
557
|
"""Set the masking function on the background logger."""
|
|
563
558
|
self.global_bg_logger().set_masking_function(masking_function)
|
|
564
559
|
|
|
@@ -566,7 +561,7 @@ class BraintrustState:
|
|
|
566
561
|
_state: BraintrustState = None # type: ignore
|
|
567
562
|
|
|
568
563
|
|
|
569
|
-
_http_adapter:
|
|
564
|
+
_http_adapter: HTTPAdapter | None = None
|
|
570
565
|
|
|
571
566
|
|
|
572
567
|
def set_http_adapter(adapter: HTTPAdapter) -> None:
|
|
@@ -632,7 +627,7 @@ class RetryRequestExceptionsAdapter(HTTPAdapter):
|
|
|
632
627
|
|
|
633
628
|
|
|
634
629
|
class HTTPConnection:
|
|
635
|
-
def __init__(self, base_url: str, adapter:
|
|
630
|
+
def __init__(self, base_url: str, adapter: HTTPAdapter | None = None):
|
|
636
631
|
self.base_url = base_url
|
|
637
632
|
self.token = None
|
|
638
633
|
self.adapter = adapter
|
|
@@ -661,7 +656,7 @@ class HTTPConnection:
|
|
|
661
656
|
self.token = token
|
|
662
657
|
self._set_session_token()
|
|
663
658
|
|
|
664
|
-
def _set_adapter(self, adapter:
|
|
659
|
+
def _set_adapter(self, adapter: HTTPAdapter | None) -> None:
|
|
665
660
|
self.adapter = adapter
|
|
666
661
|
|
|
667
662
|
def _reset(self, **retry_kwargs: Any) -> None:
|
|
@@ -693,9 +688,7 @@ class HTTPConnection:
|
|
|
693
688
|
def delete(self, path: str, *args: Any, **kwargs: Any) -> requests.Response:
|
|
694
689
|
return self.session.delete(_urljoin(self.base_url, path), *args, **kwargs)
|
|
695
690
|
|
|
696
|
-
def get_json(
|
|
697
|
-
self, object_type: str, args: Optional[Mapping[str, Any]] = None, retries: int = 0
|
|
698
|
-
) -> Mapping[str, Any]:
|
|
691
|
+
def get_json(self, object_type: str, args: Mapping[str, Any] | None = None, retries: int = 0) -> Mapping[str, Any]:
|
|
699
692
|
tries = retries + 1
|
|
700
693
|
for i in range(tries):
|
|
701
694
|
resp = self.get(f"/{object_type}", params=args)
|
|
@@ -708,7 +701,7 @@ class HTTPConnection:
|
|
|
708
701
|
# Needed for type checking.
|
|
709
702
|
raise Exception("unreachable")
|
|
710
703
|
|
|
711
|
-
def post_json(self, object_type: str, args:
|
|
704
|
+
def post_json(self, object_type: str, args: Mapping[str, Any] | None = None) -> Any:
|
|
712
705
|
resp = self.post(f"/{object_type.lstrip('/')}", json=args)
|
|
713
706
|
response_raise_for_status(resp)
|
|
714
707
|
return resp.json()
|
|
@@ -749,13 +742,6 @@ def construct_logs3_data(items: Sequence[str]):
|
|
|
749
742
|
return '{"rows": ' + rowsS + ', "api_version": ' + str(DATA_API_VERSION) + "}"
|
|
750
743
|
|
|
751
744
|
|
|
752
|
-
def _check_json_serializable(event):
|
|
753
|
-
try:
|
|
754
|
-
return bt_dumps(event)
|
|
755
|
-
except (TypeError, ValueError) as e:
|
|
756
|
-
raise Exception(f"All logged values must be JSON-serializable: {event}") from e
|
|
757
|
-
|
|
758
|
-
|
|
759
745
|
class _MaskingError:
|
|
760
746
|
"""Internal class to signal masking errors that need special handling."""
|
|
761
747
|
|
|
@@ -792,11 +778,11 @@ def _apply_masking_to_field(masking_function: Callable[[Any], Any], data: Any, f
|
|
|
792
778
|
|
|
793
779
|
class _BackgroundLogger(ABC):
|
|
794
780
|
@abstractmethod
|
|
795
|
-
def log(self, *args: LazyValue[
|
|
781
|
+
def log(self, *args: LazyValue[dict[str, Any]]) -> None:
|
|
796
782
|
pass
|
|
797
783
|
|
|
798
784
|
@abstractmethod
|
|
799
|
-
def flush(self, batch_size:
|
|
785
|
+
def flush(self, batch_size: int | None = None):
|
|
800
786
|
pass
|
|
801
787
|
|
|
802
788
|
|
|
@@ -804,21 +790,36 @@ class _MemoryBackgroundLogger(_BackgroundLogger):
|
|
|
804
790
|
def __init__(self):
|
|
805
791
|
self.lock = threading.Lock()
|
|
806
792
|
self.logs = []
|
|
807
|
-
self.masking_function:
|
|
793
|
+
self.masking_function: Callable[[Any], Any] | None = None
|
|
794
|
+
self.upload_attempts: list[BaseAttachment] = [] # Track upload attempts
|
|
808
795
|
|
|
809
796
|
def enforce_queue_size_limit(self, enforce: bool) -> None:
|
|
810
797
|
pass
|
|
811
798
|
|
|
812
|
-
def log(self, *args: LazyValue[
|
|
799
|
+
def log(self, *args: LazyValue[dict[str, Any]]) -> None:
|
|
813
800
|
with self.lock:
|
|
814
801
|
self.logs.extend(args)
|
|
815
802
|
|
|
816
|
-
def set_masking_function(self, masking_function:
|
|
803
|
+
def set_masking_function(self, masking_function: Callable[[Any], Any] | None) -> None:
|
|
817
804
|
"""Set the masking function for the memory logger."""
|
|
818
805
|
self.masking_function = masking_function
|
|
819
806
|
|
|
820
|
-
def flush(self, batch_size:
|
|
821
|
-
|
|
807
|
+
def flush(self, batch_size: int | None = None):
|
|
808
|
+
"""Flush the memory logger, extracting attachments and tracking upload attempts."""
|
|
809
|
+
with self.lock:
|
|
810
|
+
if not self.logs:
|
|
811
|
+
return
|
|
812
|
+
|
|
813
|
+
# Unwrap lazy values and extract attachments
|
|
814
|
+
logs = [l.get() for l in self.logs]
|
|
815
|
+
|
|
816
|
+
# Extract attachments from all logs
|
|
817
|
+
attachments: list[BaseAttachment] = []
|
|
818
|
+
for log in logs:
|
|
819
|
+
_extract_attachments(log, attachments)
|
|
820
|
+
|
|
821
|
+
# Track upload attempts (don't actually call upload() in tests)
|
|
822
|
+
self.upload_attempts.extend(attachments)
|
|
822
823
|
|
|
823
824
|
def pop(self):
|
|
824
825
|
with self.lock:
|
|
@@ -871,7 +872,7 @@ BACKGROUND_LOGGER_BASE_SLEEP_TIME_S = 1.0
|
|
|
871
872
|
class _HTTPBackgroundLogger:
|
|
872
873
|
def __init__(self, api_conn: LazyValue[HTTPConnection]):
|
|
873
874
|
self.api_conn = api_conn
|
|
874
|
-
self.masking_function:
|
|
875
|
+
self.masking_function: Callable[[Any], Any] | None = None
|
|
875
876
|
self.outfile = sys.stderr
|
|
876
877
|
self.flush_lock = threading.RLock()
|
|
877
878
|
|
|
@@ -934,7 +935,7 @@ class _HTTPBackgroundLogger:
|
|
|
934
935
|
"""
|
|
935
936
|
self.queue.enforce_queue_size_limit(enforce)
|
|
936
937
|
|
|
937
|
-
def log(self, *args: LazyValue[
|
|
938
|
+
def log(self, *args: LazyValue[dict[str, Any]]) -> None:
|
|
938
939
|
self._start()
|
|
939
940
|
dropped_items = []
|
|
940
941
|
for event in args:
|
|
@@ -981,7 +982,7 @@ class _HTTPBackgroundLogger:
|
|
|
981
982
|
else:
|
|
982
983
|
raise
|
|
983
984
|
|
|
984
|
-
def flush(self, batch_size:
|
|
985
|
+
def flush(self, batch_size: int | None = None):
|
|
985
986
|
if batch_size is None:
|
|
986
987
|
batch_size = self.default_batch_size
|
|
987
988
|
|
|
@@ -1020,7 +1021,7 @@ class _HTTPBackgroundLogger:
|
|
|
1020
1021
|
f"Encountered the following errors while logging:", post_promise_exceptions
|
|
1021
1022
|
)
|
|
1022
1023
|
|
|
1023
|
-
attachment_errors:
|
|
1024
|
+
attachment_errors: list[Exception] = []
|
|
1024
1025
|
for attachment in attachments:
|
|
1025
1026
|
try:
|
|
1026
1027
|
result = attachment.upload()
|
|
@@ -1038,8 +1039,8 @@ class _HTTPBackgroundLogger:
|
|
|
1038
1039
|
)
|
|
1039
1040
|
|
|
1040
1041
|
def _unwrap_lazy_values(
|
|
1041
|
-
self, wrapped_items: Sequence[LazyValue[
|
|
1042
|
-
) ->
|
|
1042
|
+
self, wrapped_items: Sequence[LazyValue[dict[str, Any]]]
|
|
1043
|
+
) -> tuple[list[list[dict[str, Any]]], list["BaseAttachment"]]:
|
|
1043
1044
|
for i in range(self.num_tries):
|
|
1044
1045
|
try:
|
|
1045
1046
|
unwrapped_items = [item.get() for item in wrapped_items]
|
|
@@ -1069,7 +1070,7 @@ class _HTTPBackgroundLogger:
|
|
|
1069
1070
|
|
|
1070
1071
|
batched_items[batch_idx][item_idx] = masked_item
|
|
1071
1072
|
|
|
1072
|
-
attachments:
|
|
1073
|
+
attachments: list["BaseAttachment"] = []
|
|
1073
1074
|
for batch in batched_items:
|
|
1074
1075
|
for item in batch:
|
|
1075
1076
|
_extract_attachments(item, attachments)
|
|
@@ -1179,7 +1180,7 @@ class _HTTPBackgroundLogger:
|
|
|
1179
1180
|
def internal_replace_api_conn(self, api_conn: HTTPConnection):
|
|
1180
1181
|
self.api_conn = LazyValue(lambda: api_conn, use_mutex=False)
|
|
1181
1182
|
|
|
1182
|
-
def set_masking_function(self, masking_function:
|
|
1183
|
+
def set_masking_function(self, masking_function: Callable[[Any], Any] | None):
|
|
1183
1184
|
"""Set or update the masking function."""
|
|
1184
1185
|
self.masking_function = masking_function
|
|
1185
1186
|
|
|
@@ -1221,7 +1222,7 @@ def _internal_with_memory_background_logger():
|
|
|
1221
1222
|
class ObjectMetadata:
|
|
1222
1223
|
id: str
|
|
1223
1224
|
name: str
|
|
1224
|
-
full_info:
|
|
1225
|
+
full_info: dict[str, Any]
|
|
1225
1226
|
|
|
1226
1227
|
|
|
1227
1228
|
@dataclasses.dataclass
|
|
@@ -1250,69 +1251,69 @@ class OrgProjectMetadata:
|
|
|
1250
1251
|
# this.
|
|
1251
1252
|
@overload
|
|
1252
1253
|
def init(
|
|
1253
|
-
project:
|
|
1254
|
-
experiment:
|
|
1255
|
-
description:
|
|
1254
|
+
project: str | None = ...,
|
|
1255
|
+
experiment: str | None = ...,
|
|
1256
|
+
description: str | None = ...,
|
|
1256
1257
|
dataset: Optional["Dataset"] = ...,
|
|
1257
1258
|
open: Literal[False] = ...,
|
|
1258
|
-
base_experiment:
|
|
1259
|
+
base_experiment: str | None = ...,
|
|
1259
1260
|
is_public: bool = ...,
|
|
1260
|
-
app_url:
|
|
1261
|
-
api_key:
|
|
1262
|
-
org_name:
|
|
1263
|
-
metadata:
|
|
1264
|
-
git_metadata_settings:
|
|
1261
|
+
app_url: str | None = ...,
|
|
1262
|
+
api_key: str | None = ...,
|
|
1263
|
+
org_name: str | None = ...,
|
|
1264
|
+
metadata: Metadata | None = ...,
|
|
1265
|
+
git_metadata_settings: GitMetadataSettings | None = ...,
|
|
1265
1266
|
set_current: bool = ...,
|
|
1266
|
-
update:
|
|
1267
|
-
project_id:
|
|
1268
|
-
base_experiment_id:
|
|
1269
|
-
repo_info:
|
|
1270
|
-
state:
|
|
1267
|
+
update: bool | None = ...,
|
|
1268
|
+
project_id: str | None = ...,
|
|
1269
|
+
base_experiment_id: str | None = ...,
|
|
1270
|
+
repo_info: RepoInfo | None = ...,
|
|
1271
|
+
state: BraintrustState | None = ...,
|
|
1271
1272
|
) -> "Experiment": ...
|
|
1272
1273
|
|
|
1273
1274
|
|
|
1274
1275
|
@overload
|
|
1275
1276
|
def init(
|
|
1276
|
-
project:
|
|
1277
|
-
experiment:
|
|
1278
|
-
description:
|
|
1277
|
+
project: str | None = ...,
|
|
1278
|
+
experiment: str | None = ...,
|
|
1279
|
+
description: str | None = ...,
|
|
1279
1280
|
dataset: Optional["Dataset"] = ...,
|
|
1280
1281
|
open: Literal[True] = ...,
|
|
1281
|
-
base_experiment:
|
|
1282
|
+
base_experiment: str | None = ...,
|
|
1282
1283
|
is_public: bool = ...,
|
|
1283
|
-
app_url:
|
|
1284
|
-
api_key:
|
|
1285
|
-
org_name:
|
|
1286
|
-
metadata:
|
|
1287
|
-
git_metadata_settings:
|
|
1284
|
+
app_url: str | None = ...,
|
|
1285
|
+
api_key: str | None = ...,
|
|
1286
|
+
org_name: str | None = ...,
|
|
1287
|
+
metadata: Metadata | None = ...,
|
|
1288
|
+
git_metadata_settings: GitMetadataSettings | None = ...,
|
|
1288
1289
|
set_current: bool = ...,
|
|
1289
|
-
update:
|
|
1290
|
-
project_id:
|
|
1291
|
-
base_experiment_id:
|
|
1292
|
-
repo_info:
|
|
1293
|
-
state:
|
|
1290
|
+
update: bool | None = ...,
|
|
1291
|
+
project_id: str | None = ...,
|
|
1292
|
+
base_experiment_id: str | None = ...,
|
|
1293
|
+
repo_info: RepoInfo | None = ...,
|
|
1294
|
+
state: BraintrustState | None = ...,
|
|
1294
1295
|
) -> "ReadonlyExperiment": ...
|
|
1295
1296
|
|
|
1296
1297
|
|
|
1297
1298
|
def init(
|
|
1298
|
-
project:
|
|
1299
|
-
experiment:
|
|
1300
|
-
description:
|
|
1299
|
+
project: str | None = None,
|
|
1300
|
+
experiment: str | None = None,
|
|
1301
|
+
description: str | None = None,
|
|
1301
1302
|
dataset: Optional["Dataset"] = None,
|
|
1302
1303
|
open: bool = False,
|
|
1303
|
-
base_experiment:
|
|
1304
|
+
base_experiment: str | None = None,
|
|
1304
1305
|
is_public: bool = False,
|
|
1305
|
-
app_url:
|
|
1306
|
-
api_key:
|
|
1307
|
-
org_name:
|
|
1308
|
-
metadata:
|
|
1309
|
-
git_metadata_settings:
|
|
1306
|
+
app_url: str | None = None,
|
|
1307
|
+
api_key: str | None = None,
|
|
1308
|
+
org_name: str | None = None,
|
|
1309
|
+
metadata: Metadata | None = None,
|
|
1310
|
+
git_metadata_settings: GitMetadataSettings | None = None,
|
|
1310
1311
|
set_current: bool = True,
|
|
1311
|
-
update:
|
|
1312
|
-
project_id:
|
|
1313
|
-
base_experiment_id:
|
|
1314
|
-
repo_info:
|
|
1315
|
-
state:
|
|
1312
|
+
update: bool | None = None,
|
|
1313
|
+
project_id: str | None = None,
|
|
1314
|
+
base_experiment_id: str | None = None,
|
|
1315
|
+
repo_info: RepoInfo | None = None,
|
|
1316
|
+
state: BraintrustState | None = None,
|
|
1316
1317
|
) -> Union["Experiment", "ReadonlyExperiment"]:
|
|
1317
1318
|
"""
|
|
1318
1319
|
Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
|
|
@@ -1460,18 +1461,18 @@ def init_experiment(*args, **kwargs) -> Union["Experiment", "ReadonlyExperiment"
|
|
|
1460
1461
|
|
|
1461
1462
|
|
|
1462
1463
|
def init_dataset(
|
|
1463
|
-
project:
|
|
1464
|
-
name:
|
|
1465
|
-
description:
|
|
1466
|
-
version:
|
|
1467
|
-
app_url:
|
|
1468
|
-
api_key:
|
|
1469
|
-
org_name:
|
|
1470
|
-
project_id:
|
|
1471
|
-
metadata:
|
|
1464
|
+
project: str | None = None,
|
|
1465
|
+
name: str | None = None,
|
|
1466
|
+
description: str | None = None,
|
|
1467
|
+
version: str | int | None = None,
|
|
1468
|
+
app_url: str | None = None,
|
|
1469
|
+
api_key: str | None = None,
|
|
1470
|
+
org_name: str | None = None,
|
|
1471
|
+
project_id: str | None = None,
|
|
1472
|
+
metadata: Metadata | None = None,
|
|
1472
1473
|
use_output: bool = DEFAULT_IS_LEGACY_DATASET,
|
|
1473
|
-
_internal_btql:
|
|
1474
|
-
state:
|
|
1474
|
+
_internal_btql: dict[str, Any] | None = None,
|
|
1475
|
+
state: BraintrustState | None = None,
|
|
1475
1476
|
) -> "Dataset":
|
|
1476
1477
|
"""
|
|
1477
1478
|
Create a new dataset in a specified project. If the project does not exist, it will be created.
|
|
@@ -1519,7 +1520,7 @@ def init_dataset(
|
|
|
1519
1520
|
)
|
|
1520
1521
|
|
|
1521
1522
|
|
|
1522
|
-
def _compute_logger_metadata(project_name:
|
|
1523
|
+
def _compute_logger_metadata(project_name: str | None = None, project_id: str | None = None):
|
|
1523
1524
|
login()
|
|
1524
1525
|
org_id = _state.org_id
|
|
1525
1526
|
if project_id is None:
|
|
@@ -1547,15 +1548,15 @@ def _compute_logger_metadata(project_name: Optional[str] = None, project_id: Opt
|
|
|
1547
1548
|
|
|
1548
1549
|
|
|
1549
1550
|
def init_logger(
|
|
1550
|
-
project:
|
|
1551
|
-
project_id:
|
|
1551
|
+
project: str | None = None,
|
|
1552
|
+
project_id: str | None = None,
|
|
1552
1553
|
async_flush: bool = True,
|
|
1553
|
-
app_url:
|
|
1554
|
-
api_key:
|
|
1555
|
-
org_name:
|
|
1554
|
+
app_url: str | None = None,
|
|
1555
|
+
api_key: str | None = None,
|
|
1556
|
+
org_name: str | None = None,
|
|
1556
1557
|
force_login: bool = False,
|
|
1557
1558
|
set_current: bool = True,
|
|
1558
|
-
state:
|
|
1559
|
+
state: BraintrustState | None = None,
|
|
1559
1560
|
) -> "Logger":
|
|
1560
1561
|
"""
|
|
1561
1562
|
Create a new logger in a specified project. If the project does not exist, it will be created.
|
|
@@ -1604,17 +1605,17 @@ def init_logger(
|
|
|
1604
1605
|
|
|
1605
1606
|
|
|
1606
1607
|
def load_prompt(
|
|
1607
|
-
project:
|
|
1608
|
-
slug:
|
|
1609
|
-
version:
|
|
1610
|
-
project_id:
|
|
1611
|
-
id:
|
|
1612
|
-
defaults:
|
|
1608
|
+
project: str | None = None,
|
|
1609
|
+
slug: str | None = None,
|
|
1610
|
+
version: str | int | None = None,
|
|
1611
|
+
project_id: str | None = None,
|
|
1612
|
+
id: str | None = None,
|
|
1613
|
+
defaults: Mapping[str, Any] | None = None,
|
|
1613
1614
|
no_trace: bool = False,
|
|
1614
|
-
environment:
|
|
1615
|
-
app_url:
|
|
1616
|
-
api_key:
|
|
1617
|
-
org_name:
|
|
1615
|
+
environment: str | None = None,
|
|
1616
|
+
app_url: str | None = None,
|
|
1617
|
+
api_key: str | None = None,
|
|
1618
|
+
org_name: str | None = None,
|
|
1618
1619
|
) -> "Prompt":
|
|
1619
1620
|
"""
|
|
1620
1621
|
Loads a prompt from the specified project.
|
|
@@ -1737,9 +1738,9 @@ login_lock = threading.RLock()
|
|
|
1737
1738
|
|
|
1738
1739
|
|
|
1739
1740
|
def login(
|
|
1740
|
-
app_url:
|
|
1741
|
-
api_key:
|
|
1742
|
-
org_name:
|
|
1741
|
+
app_url: str | None = None,
|
|
1742
|
+
api_key: str | None = None,
|
|
1743
|
+
org_name: str | None = None,
|
|
1743
1744
|
force_login: bool = False,
|
|
1744
1745
|
) -> None:
|
|
1745
1746
|
"""
|
|
@@ -1763,9 +1764,9 @@ def login(
|
|
|
1763
1764
|
|
|
1764
1765
|
|
|
1765
1766
|
def login_to_state(
|
|
1766
|
-
app_url:
|
|
1767
|
-
api_key:
|
|
1768
|
-
org_name:
|
|
1767
|
+
app_url: str | None = None,
|
|
1768
|
+
api_key: str | None = None,
|
|
1769
|
+
org_name: str | None = None,
|
|
1769
1770
|
) -> BraintrustState:
|
|
1770
1771
|
app_url = _get_app_url(app_url)
|
|
1771
1772
|
|
|
@@ -1845,7 +1846,7 @@ def login_to_state(
|
|
|
1845
1846
|
return state
|
|
1846
1847
|
|
|
1847
1848
|
|
|
1848
|
-
def set_masking_function(masking_function:
|
|
1849
|
+
def set_masking_function(masking_function: Callable[[Any], Any] | None) -> None:
|
|
1849
1850
|
"""
|
|
1850
1851
|
Set a global masking function that will be applied to all logged data before sending to Braintrust.
|
|
1851
1852
|
The masking function will be applied after records are merged but before they are sent to the backend.
|
|
@@ -1872,7 +1873,7 @@ def log(**event: Any) -> str:
|
|
|
1872
1873
|
return e.log(**event)
|
|
1873
1874
|
|
|
1874
1875
|
|
|
1875
|
-
def summarize(summarize_scores: bool = True, comparison_experiment_id:
|
|
1876
|
+
def summarize(summarize_scores: bool = True, comparison_experiment_id: str | None = None) -> "ExperimentSummary":
|
|
1876
1877
|
"""
|
|
1877
1878
|
Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
1878
1879
|
|
|
@@ -1918,7 +1919,7 @@ def current_span() -> Span:
|
|
|
1918
1919
|
|
|
1919
1920
|
|
|
1920
1921
|
@contextlib.contextmanager
|
|
1921
|
-
def parent_context(parent:
|
|
1922
|
+
def parent_context(parent: str | None, state: BraintrustState | None = None):
|
|
1922
1923
|
"""
|
|
1923
1924
|
Context manager to temporarily set the parent context for spans.
|
|
1924
1925
|
|
|
@@ -1940,7 +1941,7 @@ def parent_context(parent: Optional[str], state: Optional[BraintrustState] = Non
|
|
|
1940
1941
|
|
|
1941
1942
|
|
|
1942
1943
|
def get_span_parent_object(
|
|
1943
|
-
parent:
|
|
1944
|
+
parent: str | None = None, state: BraintrustState | None = None
|
|
1944
1945
|
) -> Union[SpanComponentsV4, "Logger", "Experiment", Span]:
|
|
1945
1946
|
"""Mainly for internal use. Return the parent object for starting a span in a global context.
|
|
1946
1947
|
Applies precedence: current span > propagated parent string > experiment > logger."""
|
|
@@ -1969,24 +1970,14 @@ def get_span_parent_object(
|
|
|
1969
1970
|
|
|
1970
1971
|
def _try_log_input(span, f_sig, f_args, f_kwargs):
|
|
1971
1972
|
if f_sig:
|
|
1972
|
-
|
|
1973
|
-
input_serializable = bound_args
|
|
1973
|
+
input_data = f_sig.bind(*f_args, **f_kwargs).arguments
|
|
1974
1974
|
else:
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
_check_json_serializable(input_serializable)
|
|
1978
|
-
except Exception as e:
|
|
1979
|
-
input_serializable = "<input not json-serializable>: " + str(e)
|
|
1980
|
-
span.log(input=input_serializable)
|
|
1975
|
+
input_data = dict(args=f_args, kwargs=f_kwargs)
|
|
1976
|
+
span.log(input=input_data)
|
|
1981
1977
|
|
|
1982
1978
|
|
|
1983
1979
|
def _try_log_output(span, output):
|
|
1984
|
-
|
|
1985
|
-
try:
|
|
1986
|
-
_check_json_serializable(output)
|
|
1987
|
-
except Exception as e:
|
|
1988
|
-
output_serializable = "<output not json-serializable>: " + str(e)
|
|
1989
|
-
span.log(output=output_serializable)
|
|
1980
|
+
span.log(output=output)
|
|
1990
1981
|
|
|
1991
1982
|
|
|
1992
1983
|
F = TypeVar("F", bound=Callable[..., Any])
|
|
@@ -2155,14 +2146,14 @@ def traced(*span_args: Any, **span_kwargs: Any) -> Callable[[F], F]:
|
|
|
2155
2146
|
|
|
2156
2147
|
|
|
2157
2148
|
def start_span(
|
|
2158
|
-
name:
|
|
2159
|
-
type:
|
|
2160
|
-
span_attributes:
|
|
2161
|
-
start_time:
|
|
2162
|
-
set_current:
|
|
2163
|
-
parent:
|
|
2164
|
-
propagated_event:
|
|
2165
|
-
state:
|
|
2149
|
+
name: str | None = None,
|
|
2150
|
+
type: SpanTypeAttribute | None = None,
|
|
2151
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
2152
|
+
start_time: float | None = None,
|
|
2153
|
+
set_current: bool | None = None,
|
|
2154
|
+
parent: str | None = None,
|
|
2155
|
+
propagated_event: dict[str, Any] | None = None,
|
|
2156
|
+
state: BraintrustState | None = None,
|
|
2166
2157
|
**event: Any,
|
|
2167
2158
|
) -> Span:
|
|
2168
2159
|
"""Lower-level alternative to `@traced` for starting a span at the toplevel. It creates a span under the first active object (using the same precedence order as `@traced`), or if `parent` is specified, under the specified parent row, or returns a no-op span object.
|
|
@@ -2265,7 +2256,7 @@ def validate_tags(tags: Sequence[str]) -> None:
|
|
|
2265
2256
|
seen.add(tag)
|
|
2266
2257
|
|
|
2267
2258
|
|
|
2268
|
-
def _extract_attachments(event:
|
|
2259
|
+
def _extract_attachments(event: dict[str, Any], attachments: list["BaseAttachment"]) -> None:
|
|
2269
2260
|
"""
|
|
2270
2261
|
Helper function for uploading attachments. Recursively extracts `Attachment`
|
|
2271
2262
|
and `ExternalAttachment` values and replaces them with their associated
|
|
@@ -2282,13 +2273,13 @@ def _extract_attachments(event: Dict[str, Any], attachments: List["BaseAttachmen
|
|
|
2282
2273
|
return v.reference # Attachment cannot be nested.
|
|
2283
2274
|
|
|
2284
2275
|
# Recursive case: object.
|
|
2285
|
-
if isinstance(v,
|
|
2276
|
+
if isinstance(v, dict):
|
|
2286
2277
|
for k, v2 in v.items():
|
|
2287
2278
|
v[k] = _helper(v2)
|
|
2288
2279
|
return v
|
|
2289
2280
|
|
|
2290
2281
|
# Recursive case: array.
|
|
2291
|
-
if isinstance(v,
|
|
2282
|
+
if isinstance(v, list):
|
|
2292
2283
|
for i in range(len(v)):
|
|
2293
2284
|
v[i] = _helper(v[i])
|
|
2294
2285
|
return v
|
|
@@ -2308,7 +2299,7 @@ def _enrich_attachments(event: TMutableMapping) -> TMutableMapping:
|
|
|
2308
2299
|
"""
|
|
2309
2300
|
|
|
2310
2301
|
def _helper(v: Any) -> Any:
|
|
2311
|
-
if isinstance(v,
|
|
2302
|
+
if isinstance(v, dict):
|
|
2312
2303
|
# Base case: AttachmentReference.
|
|
2313
2304
|
if v.get("type") == "braintrust_attachment" or v.get("type") == "external_attachment":
|
|
2314
2305
|
return ReadonlyAttachment(cast(AttachmentReference, v))
|
|
@@ -2319,7 +2310,7 @@ def _enrich_attachments(event: TMutableMapping) -> TMutableMapping:
|
|
|
2319
2310
|
return v
|
|
2320
2311
|
|
|
2321
2312
|
# Recursive case: array.
|
|
2322
|
-
if isinstance(v,
|
|
2313
|
+
if isinstance(v, list):
|
|
2323
2314
|
for i in range(len(v)):
|
|
2324
2315
|
v[i] = _helper(v[i])
|
|
2325
2316
|
return v
|
|
@@ -2333,7 +2324,7 @@ def _enrich_attachments(event: TMutableMapping) -> TMutableMapping:
|
|
|
2333
2324
|
return event
|
|
2334
2325
|
|
|
2335
2326
|
|
|
2336
|
-
def _validate_and_sanitize_experiment_log_partial_args(event: Mapping[str, Any]) ->
|
|
2327
|
+
def _validate_and_sanitize_experiment_log_partial_args(event: Mapping[str, Any]) -> dict[str, Any]:
|
|
2337
2328
|
# Make sure only certain keys are specified.
|
|
2338
2329
|
forbidden_keys = set(event.keys()) - {
|
|
2339
2330
|
"input",
|
|
@@ -2436,91 +2427,6 @@ def _validate_and_sanitize_experiment_log_full_args(event: Mapping[str, Any], ha
|
|
|
2436
2427
|
return event
|
|
2437
2428
|
|
|
2438
2429
|
|
|
2439
|
-
def _deep_copy_event(event: Mapping[str, Any]) -> Dict[str, Any]:
|
|
2440
|
-
"""
|
|
2441
|
-
Creates a deep copy of the given event. Replaces references to user objects
|
|
2442
|
-
with placeholder strings to ensure serializability, except for `Attachment`
|
|
2443
|
-
and `ExternalAttachment` objects, which are preserved and not deep-copied.
|
|
2444
|
-
|
|
2445
|
-
Handles circular references and excessive nesting depth to prevent
|
|
2446
|
-
RecursionError during serialization.
|
|
2447
|
-
"""
|
|
2448
|
-
# Maximum depth to prevent hitting Python's recursion limit
|
|
2449
|
-
# Python's default limit is ~1000, we use a conservative limit
|
|
2450
|
-
# to account for existing call stack usage from pytest, application code, etc.
|
|
2451
|
-
MAX_DEPTH = 200
|
|
2452
|
-
|
|
2453
|
-
# Track visited objects to detect circular references
|
|
2454
|
-
visited: set[int] = set()
|
|
2455
|
-
|
|
2456
|
-
def _deep_copy_object(v: Any, depth: int = 0) -> Any:
|
|
2457
|
-
# Check depth limit - use >= to stop before exceeding
|
|
2458
|
-
if depth >= MAX_DEPTH:
|
|
2459
|
-
return "<max depth exceeded>"
|
|
2460
|
-
|
|
2461
|
-
# Check for circular references in mutable containers
|
|
2462
|
-
# Use id() to track object identity
|
|
2463
|
-
if isinstance(v, (Mapping, List, Tuple, Set)):
|
|
2464
|
-
obj_id = id(v)
|
|
2465
|
-
if obj_id in visited:
|
|
2466
|
-
return "<circular reference>"
|
|
2467
|
-
visited.add(obj_id)
|
|
2468
|
-
try:
|
|
2469
|
-
if isinstance(v, Mapping):
|
|
2470
|
-
# Prevent dict keys from holding references to user data. Note that
|
|
2471
|
-
# `bt_json` already coerces keys to string, a behavior that comes from
|
|
2472
|
-
# `json.dumps`. However, that runs at log upload time, while we want to
|
|
2473
|
-
# cut out all the references to user objects synchronously in this
|
|
2474
|
-
# function.
|
|
2475
|
-
result = {}
|
|
2476
|
-
for k in v:
|
|
2477
|
-
try:
|
|
2478
|
-
key_str = str(k)
|
|
2479
|
-
except Exception:
|
|
2480
|
-
# If str() fails on the key, use a fallback representation
|
|
2481
|
-
key_str = f"<non-stringifiable-key: {type(k).__name__}>"
|
|
2482
|
-
result[key_str] = _deep_copy_object(v[k], depth + 1)
|
|
2483
|
-
return result
|
|
2484
|
-
elif isinstance(v, (List, Tuple, Set)):
|
|
2485
|
-
return [_deep_copy_object(x, depth + 1) for x in v]
|
|
2486
|
-
finally:
|
|
2487
|
-
# Remove from visited set after processing to allow the same object
|
|
2488
|
-
# to appear in different branches of the tree
|
|
2489
|
-
visited.discard(obj_id)
|
|
2490
|
-
|
|
2491
|
-
if isinstance(v, Span):
|
|
2492
|
-
return "<span>"
|
|
2493
|
-
elif isinstance(v, Experiment):
|
|
2494
|
-
return "<experiment>"
|
|
2495
|
-
elif isinstance(v, Dataset):
|
|
2496
|
-
return "<dataset>"
|
|
2497
|
-
elif isinstance(v, Logger):
|
|
2498
|
-
return "<logger>"
|
|
2499
|
-
elif isinstance(v, BaseAttachment):
|
|
2500
|
-
return v
|
|
2501
|
-
elif isinstance(v, ReadonlyAttachment):
|
|
2502
|
-
return v.reference
|
|
2503
|
-
elif isinstance(v, float):
|
|
2504
|
-
# Handle NaN and Infinity for JSON compatibility
|
|
2505
|
-
if math.isnan(v):
|
|
2506
|
-
return "NaN"
|
|
2507
|
-
elif math.isinf(v):
|
|
2508
|
-
return "Infinity" if v > 0 else "-Infinity"
|
|
2509
|
-
return v
|
|
2510
|
-
elif isinstance(v, (int, str, bool)) or v is None:
|
|
2511
|
-
# Skip roundtrip for primitive types.
|
|
2512
|
-
return v
|
|
2513
|
-
else:
|
|
2514
|
-
# Note: we avoid using copy.deepcopy, because it's difficult to
|
|
2515
|
-
# guarantee the independence of such copied types from their origin.
|
|
2516
|
-
# E.g. the original type could have a `__del__` method that alters
|
|
2517
|
-
# some shared internal state, and we need this deep copy to be
|
|
2518
|
-
# fully-independent from the original.
|
|
2519
|
-
return bt_loads(bt_dumps(v))
|
|
2520
|
-
|
|
2521
|
-
return _deep_copy_object(event)
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
2430
|
class ObjectIterator(Generic[T]):
|
|
2525
2431
|
def __init__(self, refetch_fn: Callable[[], Sequence[T]]):
|
|
2526
2432
|
self.refetch_fn = refetch_fn
|
|
@@ -2547,9 +2453,9 @@ class ObjectFetcher(ABC, Generic[TMapping]):
|
|
|
2547
2453
|
def __init__(
|
|
2548
2454
|
self,
|
|
2549
2455
|
object_type: str,
|
|
2550
|
-
pinned_version:
|
|
2551
|
-
mutate_record:
|
|
2552
|
-
_internal_btql:
|
|
2456
|
+
pinned_version: None | int | str = None,
|
|
2457
|
+
mutate_record: Callable[[TMapping], TMapping] | None = None,
|
|
2458
|
+
_internal_btql: dict[str, Any] | None = None,
|
|
2553
2459
|
):
|
|
2554
2460
|
self.object_type = object_type
|
|
2555
2461
|
|
|
@@ -2563,10 +2469,10 @@ class ObjectFetcher(ABC, Generic[TMapping]):
|
|
|
2563
2469
|
self._pinned_version = str(pinned_version) if pinned_version is not None else None
|
|
2564
2470
|
self._mutate_record = mutate_record
|
|
2565
2471
|
|
|
2566
|
-
self._fetched_data:
|
|
2472
|
+
self._fetched_data: list[TMapping] | None = None
|
|
2567
2473
|
self._internal_btql = _internal_btql
|
|
2568
2474
|
|
|
2569
|
-
def fetch(self, batch_size:
|
|
2475
|
+
def fetch(self, batch_size: int | None = None) -> Iterator[TMapping]:
|
|
2570
2476
|
"""
|
|
2571
2477
|
Fetch all records.
|
|
2572
2478
|
|
|
@@ -2601,7 +2507,7 @@ class ObjectFetcher(ABC, Generic[TMapping]):
|
|
|
2601
2507
|
@abstractmethod
|
|
2602
2508
|
def id(self) -> str: ...
|
|
2603
2509
|
|
|
2604
|
-
def _refetch(self, batch_size:
|
|
2510
|
+
def _refetch(self, batch_size: int | None = None) -> list[TMapping]:
|
|
2605
2511
|
state = self._get_state()
|
|
2606
2512
|
limit = batch_size if batch_size is not None else DEFAULT_FETCH_BATCH_SIZE
|
|
2607
2513
|
if self._fetched_data is None:
|
|
@@ -2642,7 +2548,7 @@ class ObjectFetcher(ABC, Generic[TMapping]):
|
|
|
2642
2548
|
)
|
|
2643
2549
|
response_raise_for_status(resp)
|
|
2644
2550
|
resp_json = resp.json()
|
|
2645
|
-
data = (data or []) + cast(
|
|
2551
|
+
data = (data or []) + cast(list[TMapping], resp_json["data"])
|
|
2646
2552
|
if not resp_json.get("cursor", None):
|
|
2647
2553
|
break
|
|
2648
2554
|
cursor = resp_json.get("cursor", None)
|
|
@@ -2699,7 +2605,7 @@ class Attachment(BaseAttachment):
|
|
|
2699
2605
|
def __init__(
|
|
2700
2606
|
self,
|
|
2701
2607
|
*,
|
|
2702
|
-
data:
|
|
2608
|
+
data: str | bytes | bytearray,
|
|
2703
2609
|
filename: str,
|
|
2704
2610
|
content_type: str,
|
|
2705
2611
|
):
|
|
@@ -2770,7 +2676,7 @@ class Attachment(BaseAttachment):
|
|
|
2770
2676
|
try:
|
|
2771
2677
|
data = self._data.get()
|
|
2772
2678
|
except Exception as e:
|
|
2773
|
-
raise
|
|
2679
|
+
raise OSError(f"Failed to read file: {e}") from e
|
|
2774
2680
|
|
|
2775
2681
|
signed_url = metadata.get("signedUrl")
|
|
2776
2682
|
headers = metadata.get("headers")
|
|
@@ -2823,7 +2729,7 @@ class Attachment(BaseAttachment):
|
|
|
2823
2729
|
|
|
2824
2730
|
return LazyValue(error_wrapper, use_mutex=True)
|
|
2825
2731
|
|
|
2826
|
-
def _init_data(self, data:
|
|
2732
|
+
def _init_data(self, data: str | bytes | bytearray) -> LazyValue[bytes]:
|
|
2827
2733
|
if isinstance(data, str):
|
|
2828
2734
|
self._ensure_file_readable(data)
|
|
2829
2735
|
|
|
@@ -3041,11 +2947,11 @@ def _log_feedback_impl(
|
|
|
3041
2947
|
parent_object_type: SpanObjectTypeV3,
|
|
3042
2948
|
parent_object_id: LazyValue[str],
|
|
3043
2949
|
id: str,
|
|
3044
|
-
scores:
|
|
3045
|
-
expected:
|
|
3046
|
-
tags:
|
|
3047
|
-
comment:
|
|
3048
|
-
metadata:
|
|
2950
|
+
scores: Mapping[str, int | float] | None = None,
|
|
2951
|
+
expected: Any | None = None,
|
|
2952
|
+
tags: Sequence[str] | None = None,
|
|
2953
|
+
comment: str | None = None,
|
|
2954
|
+
metadata: Mapping[str, Any] | None = None,
|
|
3049
2955
|
source: Literal["external", "app", "api", None] = None,
|
|
3050
2956
|
):
|
|
3051
2957
|
if source is None:
|
|
@@ -3070,7 +2976,7 @@ def _log_feedback_impl(
|
|
|
3070
2976
|
metadata = update_event.pop("metadata")
|
|
3071
2977
|
update_event = {k: v for k, v in update_event.items() if v is not None}
|
|
3072
2978
|
|
|
3073
|
-
update_event =
|
|
2979
|
+
update_event = bt_safe_deep_copy(update_event)
|
|
3074
2980
|
|
|
3075
2981
|
def parent_ids():
|
|
3076
2982
|
exporter = _get_exporter()
|
|
@@ -3126,7 +3032,7 @@ def _update_span_impl(
|
|
|
3126
3032
|
event=event,
|
|
3127
3033
|
)
|
|
3128
3034
|
|
|
3129
|
-
update_event =
|
|
3035
|
+
update_event = bt_safe_deep_copy(update_event)
|
|
3130
3036
|
|
|
3131
3037
|
def parent_ids():
|
|
3132
3038
|
exporter = _get_exporter()
|
|
@@ -3185,13 +3091,13 @@ class SpanIds:
|
|
|
3185
3091
|
|
|
3186
3092
|
span_id: str
|
|
3187
3093
|
root_span_id: str
|
|
3188
|
-
span_parents:
|
|
3094
|
+
span_parents: list[str] | None
|
|
3189
3095
|
|
|
3190
3096
|
|
|
3191
3097
|
def _resolve_span_ids(
|
|
3192
|
-
span_id:
|
|
3193
|
-
root_span_id:
|
|
3194
|
-
parent_span_ids:
|
|
3098
|
+
span_id: str | None,
|
|
3099
|
+
root_span_id: str | None,
|
|
3100
|
+
parent_span_ids: ParentSpanIds | None,
|
|
3195
3101
|
lookup_span_parent: bool,
|
|
3196
3102
|
id_generator: "id_gen.IDGenerator",
|
|
3197
3103
|
context_manager: "context.ContextManager",
|
|
@@ -3265,7 +3171,7 @@ def span_components_to_object_id(components: SpanComponentsV4) -> str:
|
|
|
3265
3171
|
return _span_components_to_object_id_lambda(components)()
|
|
3266
3172
|
|
|
3267
3173
|
|
|
3268
|
-
def permalink(slug: str, org_name:
|
|
3174
|
+
def permalink(slug: str, org_name: str | None = None, app_url: str | None = None) -> str:
|
|
3269
3175
|
"""
|
|
3270
3176
|
Format a permalink to the Braintrust application for viewing the span represented by the provided `slug`.
|
|
3271
3177
|
|
|
@@ -3314,13 +3220,13 @@ def permalink(slug: str, org_name: Optional[str] = None, app_url: Optional[str]
|
|
|
3314
3220
|
|
|
3315
3221
|
|
|
3316
3222
|
def _start_span_parent_args(
|
|
3317
|
-
parent:
|
|
3223
|
+
parent: str | None,
|
|
3318
3224
|
parent_object_type: SpanObjectTypeV3,
|
|
3319
3225
|
parent_object_id: LazyValue[str],
|
|
3320
|
-
parent_compute_object_metadata_args:
|
|
3321
|
-
parent_span_ids:
|
|
3322
|
-
propagated_event:
|
|
3323
|
-
) ->
|
|
3226
|
+
parent_compute_object_metadata_args: dict[str, Any] | None,
|
|
3227
|
+
parent_span_ids: ParentSpanIds | None,
|
|
3228
|
+
propagated_event: dict[str, Any] | None,
|
|
3229
|
+
) -> dict[str, Any]:
|
|
3324
3230
|
if parent:
|
|
3325
3231
|
assert parent_span_ids is None, "Cannot specify both parent and parent_span_ids"
|
|
3326
3232
|
parent_components = SpanComponentsV4.from_str(parent)
|
|
@@ -3374,9 +3280,9 @@ class _ExperimentDatasetEvent(TypedDict):
|
|
|
3374
3280
|
|
|
3375
3281
|
id: str
|
|
3376
3282
|
_xact_id: str
|
|
3377
|
-
input:
|
|
3378
|
-
expected:
|
|
3379
|
-
tags:
|
|
3283
|
+
input: Any | None
|
|
3284
|
+
expected: Any | None
|
|
3285
|
+
tags: Sequence[str] | None
|
|
3380
3286
|
|
|
3381
3287
|
|
|
3382
3288
|
class ExperimentDatasetIterator:
|
|
@@ -3422,7 +3328,7 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3422
3328
|
self,
|
|
3423
3329
|
lazy_metadata: LazyValue[ProjectExperimentMetadata],
|
|
3424
3330
|
dataset: Optional["Dataset"] = None,
|
|
3425
|
-
state:
|
|
3331
|
+
state: BraintrustState | None = None,
|
|
3426
3332
|
):
|
|
3427
3333
|
self._lazy_metadata = lazy_metadata
|
|
3428
3334
|
self.dataset = dataset
|
|
@@ -3473,16 +3379,16 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3473
3379
|
|
|
3474
3380
|
def log(
|
|
3475
3381
|
self,
|
|
3476
|
-
input:
|
|
3477
|
-
output:
|
|
3478
|
-
expected:
|
|
3479
|
-
error:
|
|
3480
|
-
tags:
|
|
3481
|
-
scores:
|
|
3482
|
-
metadata:
|
|
3483
|
-
metrics:
|
|
3484
|
-
id:
|
|
3485
|
-
dataset_record_id:
|
|
3382
|
+
input: Any | None = None,
|
|
3383
|
+
output: Any | None = None,
|
|
3384
|
+
expected: Any | None = None,
|
|
3385
|
+
error: str | None = None,
|
|
3386
|
+
tags: Sequence[str] | None = None,
|
|
3387
|
+
scores: Mapping[str, int | float] | None = None,
|
|
3388
|
+
metadata: Mapping[str, Any] | None = None,
|
|
3389
|
+
metrics: Mapping[str, int | float] | None = None,
|
|
3390
|
+
id: str | None = None,
|
|
3391
|
+
dataset_record_id: str | None = None,
|
|
3486
3392
|
allow_concurrent_with_spans: bool = False,
|
|
3487
3393
|
) -> str:
|
|
3488
3394
|
"""
|
|
@@ -3527,11 +3433,11 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3527
3433
|
def log_feedback(
|
|
3528
3434
|
self,
|
|
3529
3435
|
id: str,
|
|
3530
|
-
scores:
|
|
3531
|
-
expected:
|
|
3532
|
-
tags:
|
|
3533
|
-
comment:
|
|
3534
|
-
metadata:
|
|
3436
|
+
scores: Mapping[str, int | float] | None = None,
|
|
3437
|
+
expected: Any | None = None,
|
|
3438
|
+
tags: Sequence[str] | None = None,
|
|
3439
|
+
comment: str | None = None,
|
|
3440
|
+
metadata: Mapping[str, Any] | None = None,
|
|
3535
3441
|
source: Literal["external", "app", "api", None] = None,
|
|
3536
3442
|
) -> None:
|
|
3537
3443
|
"""
|
|
@@ -3559,13 +3465,13 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3559
3465
|
|
|
3560
3466
|
def start_span(
|
|
3561
3467
|
self,
|
|
3562
|
-
name:
|
|
3563
|
-
type:
|
|
3564
|
-
span_attributes:
|
|
3565
|
-
start_time:
|
|
3566
|
-
set_current:
|
|
3567
|
-
parent:
|
|
3568
|
-
propagated_event:
|
|
3468
|
+
name: str | None = None,
|
|
3469
|
+
type: SpanTypeAttribute | None = None,
|
|
3470
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
3471
|
+
start_time: float | None = None,
|
|
3472
|
+
set_current: bool | None = None,
|
|
3473
|
+
parent: str | None = None,
|
|
3474
|
+
propagated_event: dict[str, Any] | None = None,
|
|
3569
3475
|
**event: Any,
|
|
3570
3476
|
) -> Span:
|
|
3571
3477
|
"""Create a new toplevel span underneath the experiment. The name defaults to "root" and the span type to "eval".
|
|
@@ -3599,7 +3505,7 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3599
3505
|
**event,
|
|
3600
3506
|
)
|
|
3601
3507
|
|
|
3602
|
-
def fetch_base_experiment(self) ->
|
|
3508
|
+
def fetch_base_experiment(self) -> ExperimentIdentifier | None:
|
|
3603
3509
|
state = self._get_state()
|
|
3604
3510
|
conn = state.app_conn()
|
|
3605
3511
|
|
|
@@ -3616,7 +3522,7 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3616
3522
|
return None
|
|
3617
3523
|
|
|
3618
3524
|
def summarize(
|
|
3619
|
-
self, summarize_scores: bool = True, comparison_experiment_id:
|
|
3525
|
+
self, summarize_scores: bool = True, comparison_experiment_id: str | None = None
|
|
3620
3526
|
) -> "ExperimentSummary":
|
|
3621
3527
|
"""
|
|
3622
3528
|
Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
@@ -3703,13 +3609,13 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3703
3609
|
|
|
3704
3610
|
def _start_span_impl(
|
|
3705
3611
|
self,
|
|
3706
|
-
name:
|
|
3707
|
-
type:
|
|
3708
|
-
span_attributes:
|
|
3709
|
-
start_time:
|
|
3710
|
-
set_current:
|
|
3711
|
-
parent:
|
|
3712
|
-
propagated_event:
|
|
3612
|
+
name: str | None = None,
|
|
3613
|
+
type: SpanTypeAttribute | None = None,
|
|
3614
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
3615
|
+
start_time: float | None = None,
|
|
3616
|
+
set_current: bool | None = None,
|
|
3617
|
+
parent: str | None = None,
|
|
3618
|
+
propagated_event: dict[str, Any] | None = None,
|
|
3713
3619
|
lookup_span_parent: bool = True,
|
|
3714
3620
|
**event: Any,
|
|
3715
3621
|
) -> Span:
|
|
@@ -3739,9 +3645,9 @@ class Experiment(ObjectFetcher[ExperimentEvent], Exportable):
|
|
|
3739
3645
|
|
|
3740
3646
|
def __exit__(
|
|
3741
3647
|
self,
|
|
3742
|
-
exc_type:
|
|
3743
|
-
exc_value:
|
|
3744
|
-
traceback:
|
|
3648
|
+
exc_type: type[BaseException] | None,
|
|
3649
|
+
exc_value: BaseException | None,
|
|
3650
|
+
traceback: TracebackType | None,
|
|
3745
3651
|
) -> None:
|
|
3746
3652
|
del exc_type, exc_value, traceback
|
|
3747
3653
|
|
|
@@ -3754,7 +3660,7 @@ class ReadonlyExperiment(ObjectFetcher[ExperimentEvent]):
|
|
|
3754
3660
|
def __init__(
|
|
3755
3661
|
self,
|
|
3756
3662
|
lazy_metadata: LazyValue[ProjectExperimentMetadata],
|
|
3757
|
-
state:
|
|
3663
|
+
state: BraintrustState | None = None,
|
|
3758
3664
|
):
|
|
3759
3665
|
self._lazy_metadata = lazy_metadata
|
|
3760
3666
|
self.state = state or _state
|
|
@@ -3779,7 +3685,7 @@ class ReadonlyExperiment(ObjectFetcher[ExperimentEvent]):
|
|
|
3779
3685
|
self._lazy_metadata.get()
|
|
3780
3686
|
return self.state
|
|
3781
3687
|
|
|
3782
|
-
def as_dataset(self, batch_size:
|
|
3688
|
+
def as_dataset(self, batch_size: int | None = None) -> Iterator[_ExperimentDatasetEvent]:
|
|
3783
3689
|
"""
|
|
3784
3690
|
Return the experiment's data as a dataset iterator.
|
|
3785
3691
|
|
|
@@ -3805,19 +3711,19 @@ class SpanImpl(Span):
|
|
|
3805
3711
|
self,
|
|
3806
3712
|
parent_object_type: SpanObjectTypeV3,
|
|
3807
3713
|
parent_object_id: LazyValue[str],
|
|
3808
|
-
parent_compute_object_metadata_args:
|
|
3809
|
-
parent_span_ids:
|
|
3810
|
-
name:
|
|
3811
|
-
type:
|
|
3812
|
-
default_root_type:
|
|
3813
|
-
span_attributes:
|
|
3814
|
-
start_time:
|
|
3815
|
-
set_current:
|
|
3816
|
-
event:
|
|
3817
|
-
propagated_event:
|
|
3818
|
-
span_id:
|
|
3819
|
-
root_span_id:
|
|
3820
|
-
state:
|
|
3714
|
+
parent_compute_object_metadata_args: dict[str, Any] | None,
|
|
3715
|
+
parent_span_ids: ParentSpanIds | None,
|
|
3716
|
+
name: str | None = None,
|
|
3717
|
+
type: SpanTypeAttribute | None = None,
|
|
3718
|
+
default_root_type: SpanTypeAttribute | None = None,
|
|
3719
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
3720
|
+
start_time: float | None = None,
|
|
3721
|
+
set_current: bool | None = None,
|
|
3722
|
+
event: dict[str, Any] | None = None,
|
|
3723
|
+
propagated_event: dict[str, Any] | None = None,
|
|
3724
|
+
span_id: str | None = None,
|
|
3725
|
+
root_span_id: str | None = None,
|
|
3726
|
+
state: BraintrustState | None = None,
|
|
3821
3727
|
lookup_span_parent: bool = True,
|
|
3822
3728
|
):
|
|
3823
3729
|
if span_attributes is None:
|
|
@@ -3830,11 +3736,11 @@ class SpanImpl(Span):
|
|
|
3830
3736
|
self.state = state or _state
|
|
3831
3737
|
|
|
3832
3738
|
self.can_set_current = cast(bool, coalesce(set_current, True))
|
|
3833
|
-
self._logged_end_time:
|
|
3739
|
+
self._logged_end_time: float | None = None
|
|
3834
3740
|
|
|
3835
3741
|
# Context token for proper cleanup - used by both OTEL and Braintrust context managers
|
|
3836
3742
|
# This is set by the context manager when the span becomes active
|
|
3837
|
-
self._context_token:
|
|
3743
|
+
self._context_token: Any | None = None
|
|
3838
3744
|
|
|
3839
3745
|
self.parent_object_type = parent_object_type
|
|
3840
3746
|
self.parent_object_id = parent_object_id
|
|
@@ -3867,7 +3773,7 @@ class SpanImpl(Span):
|
|
|
3867
3773
|
_EXEC_COUNTER += 1
|
|
3868
3774
|
exec_counter = _EXEC_COUNTER
|
|
3869
3775
|
|
|
3870
|
-
internal_data:
|
|
3776
|
+
internal_data: dict[str, Any] = dict(
|
|
3871
3777
|
metrics=dict(
|
|
3872
3778
|
start=start_time or time.time(),
|
|
3873
3779
|
),
|
|
@@ -3909,9 +3815,9 @@ class SpanImpl(Span):
|
|
|
3909
3815
|
|
|
3910
3816
|
def set_attributes(
|
|
3911
3817
|
self,
|
|
3912
|
-
name:
|
|
3913
|
-
type:
|
|
3914
|
-
span_attributes:
|
|
3818
|
+
name: str | None = None,
|
|
3819
|
+
type: SpanTypeAttribute | None = None,
|
|
3820
|
+
span_attributes: Mapping[str, Any] | None = None,
|
|
3915
3821
|
) -> None:
|
|
3916
3822
|
self.log_internal(
|
|
3917
3823
|
internal_data={
|
|
@@ -3929,9 +3835,7 @@ class SpanImpl(Span):
|
|
|
3929
3835
|
def log(self, **event: Any) -> None:
|
|
3930
3836
|
return self.log_internal(event=event, internal_data=None)
|
|
3931
3837
|
|
|
3932
|
-
def log_internal(
|
|
3933
|
-
self, event: Optional[Dict[str, Any]] = None, internal_data: Optional[Dict[str, Any]] = None
|
|
3934
|
-
) -> None:
|
|
3838
|
+
def log_internal(self, event: dict[str, Any] | None = None, internal_data: dict[str, Any] | None = None) -> None:
|
|
3935
3839
|
serializable_partial_record, lazy_partial_record = split_logging_data(event, internal_data)
|
|
3936
3840
|
|
|
3937
3841
|
# We both check for serializability and round-trip `partial_record`
|
|
@@ -3939,7 +3843,7 @@ class SpanImpl(Span):
|
|
|
3939
3843
|
# cutting out any reference to user objects when the object is logged
|
|
3940
3844
|
# asynchronously, so that in case the objects are modified, the logging
|
|
3941
3845
|
# is unaffected.
|
|
3942
|
-
partial_record:
|
|
3846
|
+
partial_record: dict[str, Any] = dict(
|
|
3943
3847
|
id=self.id,
|
|
3944
3848
|
span_id=self.span_id,
|
|
3945
3849
|
root_span_id=self.root_span_id,
|
|
@@ -3948,15 +3852,14 @@ class SpanImpl(Span):
|
|
|
3948
3852
|
**{IS_MERGE_FIELD: self._is_merge},
|
|
3949
3853
|
)
|
|
3950
3854
|
|
|
3951
|
-
serializable_partial_record =
|
|
3952
|
-
_check_json_serializable(serializable_partial_record)
|
|
3855
|
+
serializable_partial_record = bt_safe_deep_copy(partial_record)
|
|
3953
3856
|
if serializable_partial_record.get("metrics", {}).get("end") is not None:
|
|
3954
3857
|
self._logged_end_time = serializable_partial_record["metrics"]["end"]
|
|
3955
3858
|
|
|
3956
3859
|
if len(serializable_partial_record.get("tags", [])) > 0 and self.span_parents:
|
|
3957
3860
|
raise Exception("Tags can only be logged to the root span")
|
|
3958
3861
|
|
|
3959
|
-
def compute_record() ->
|
|
3862
|
+
def compute_record() -> dict[str, Any]:
|
|
3960
3863
|
exporter = _get_exporter()
|
|
3961
3864
|
return dict(
|
|
3962
3865
|
**serializable_partial_record,
|
|
@@ -3979,13 +3882,13 @@ class SpanImpl(Span):
|
|
|
3979
3882
|
|
|
3980
3883
|
def start_span(
|
|
3981
3884
|
self,
|
|
3982
|
-
name:
|
|
3983
|
-
type:
|
|
3984
|
-
span_attributes:
|
|
3985
|
-
start_time:
|
|
3986
|
-
set_current:
|
|
3987
|
-
parent:
|
|
3988
|
-
propagated_event:
|
|
3885
|
+
name: str | None = None,
|
|
3886
|
+
type: SpanTypeAttribute | None = None,
|
|
3887
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
3888
|
+
start_time: float | None = None,
|
|
3889
|
+
set_current: bool | None = None,
|
|
3890
|
+
parent: str | None = None,
|
|
3891
|
+
propagated_event: dict[str, Any] | None = None,
|
|
3989
3892
|
**event: Any,
|
|
3990
3893
|
) -> Span:
|
|
3991
3894
|
if parent:
|
|
@@ -4017,7 +3920,7 @@ class SpanImpl(Span):
|
|
|
4017
3920
|
state=self.state,
|
|
4018
3921
|
)
|
|
4019
3922
|
|
|
4020
|
-
def end(self, end_time:
|
|
3923
|
+
def end(self, end_time: float | None = None) -> float:
|
|
4021
3924
|
internal_data = {}
|
|
4022
3925
|
if not self._logged_end_time:
|
|
4023
3926
|
end_time = end_time or time.time()
|
|
@@ -4162,13 +4065,13 @@ class SpanImpl(Span):
|
|
|
4162
4065
|
|
|
4163
4066
|
|
|
4164
4067
|
def log_exc_info_to_span(
|
|
4165
|
-
span: Span, exc_type:
|
|
4068
|
+
span: Span, exc_type: type[BaseException], exc_value: BaseException, tb: TracebackType | None
|
|
4166
4069
|
) -> None:
|
|
4167
4070
|
error = stringify_exception(exc_type, exc_value, tb)
|
|
4168
4071
|
span.log(error=error)
|
|
4169
4072
|
|
|
4170
4073
|
|
|
4171
|
-
def stringify_exception(exc_type:
|
|
4074
|
+
def stringify_exception(exc_type: type[BaseException], exc_value: BaseException, tb: TracebackType | None) -> str:
|
|
4172
4075
|
return "".join(
|
|
4173
4076
|
traceback.format_exception_only(exc_type, exc_value)
|
|
4174
4077
|
+ ["\nTraceback (most recent call last):\n"]
|
|
@@ -4183,8 +4086,8 @@ def _strip_nones(d: T, deep: bool) -> T:
|
|
|
4183
4086
|
|
|
4184
4087
|
|
|
4185
4088
|
def split_logging_data(
|
|
4186
|
-
event:
|
|
4187
|
-
) ->
|
|
4089
|
+
event: dict[str, Any] | None, internal_data: dict[str, Any] | None
|
|
4090
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4188
4091
|
# There should be no overlap between the dictionaries being merged,
|
|
4189
4092
|
# except for `sanitized` and `internal_data`, where the former overrides
|
|
4190
4093
|
# the latter.
|
|
@@ -4192,8 +4095,8 @@ def split_logging_data(
|
|
|
4192
4095
|
sanitized_and_internal_data = _strip_nones(internal_data or {}, deep=True)
|
|
4193
4096
|
merge_dicts(sanitized_and_internal_data, _strip_nones(sanitized, deep=False))
|
|
4194
4097
|
|
|
4195
|
-
serializable_partial_record:
|
|
4196
|
-
lazy_partial_record:
|
|
4098
|
+
serializable_partial_record: dict[str, Any] = {}
|
|
4099
|
+
lazy_partial_record: dict[str, Any] = {}
|
|
4197
4100
|
for k, v in sanitized_and_internal_data.items():
|
|
4198
4101
|
if isinstance(v, BraintrustStream):
|
|
4199
4102
|
# Python has weird semantics with loop variables and lambda functions, so we
|
|
@@ -4220,10 +4123,10 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4220
4123
|
def __init__(
|
|
4221
4124
|
self,
|
|
4222
4125
|
lazy_metadata: LazyValue[ProjectDatasetMetadata],
|
|
4223
|
-
version:
|
|
4126
|
+
version: None | int | str = None,
|
|
4224
4127
|
legacy: bool = DEFAULT_IS_LEGACY_DATASET,
|
|
4225
|
-
_internal_btql:
|
|
4226
|
-
state:
|
|
4128
|
+
_internal_btql: dict[str, Any] | None = None,
|
|
4129
|
+
state: BraintrustState | None = None,
|
|
4227
4130
|
):
|
|
4228
4131
|
if legacy:
|
|
4229
4132
|
eprint(
|
|
@@ -4231,7 +4134,7 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4231
4134
|
)
|
|
4232
4135
|
|
|
4233
4136
|
def mutate_record(r: DatasetEvent) -> DatasetEvent:
|
|
4234
|
-
_enrich_attachments(cast(
|
|
4137
|
+
_enrich_attachments(cast(dict[str, Any], r))
|
|
4235
4138
|
return ensure_dataset_record(r, legacy)
|
|
4236
4139
|
|
|
4237
4140
|
self._lazy_metadata = lazy_metadata
|
|
@@ -4278,10 +4181,10 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4278
4181
|
|
|
4279
4182
|
def _validate_event(
|
|
4280
4183
|
self,
|
|
4281
|
-
metadata:
|
|
4282
|
-
expected:
|
|
4283
|
-
output:
|
|
4284
|
-
tags:
|
|
4184
|
+
metadata: dict[str, Any] | None = None,
|
|
4185
|
+
expected: Any | None = None,
|
|
4186
|
+
output: Any | None = None,
|
|
4187
|
+
tags: Sequence[str] | None = None,
|
|
4285
4188
|
):
|
|
4286
4189
|
if metadata is not None:
|
|
4287
4190
|
if not isinstance(metadata, dict):
|
|
@@ -4298,7 +4201,7 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4298
4201
|
|
|
4299
4202
|
def _create_args(
|
|
4300
4203
|
self, id, input=None, expected=None, metadata=None, tags=None, output=None, is_merge=False
|
|
4301
|
-
) -> LazyValue[
|
|
4204
|
+
) -> LazyValue[dict[str, Any]]:
|
|
4302
4205
|
expected_value = expected if expected is not None else output
|
|
4303
4206
|
|
|
4304
4207
|
args = _populate_args(
|
|
@@ -4316,10 +4219,9 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4316
4219
|
args[IS_MERGE_FIELD] = True
|
|
4317
4220
|
args = _filter_none_args(args) # If merging, then remove None values to prevent null value writes
|
|
4318
4221
|
|
|
4319
|
-
|
|
4320
|
-
args = _deep_copy_event(args)
|
|
4222
|
+
args = bt_safe_deep_copy(args)
|
|
4321
4223
|
|
|
4322
|
-
def compute_args() ->
|
|
4224
|
+
def compute_args() -> dict[str, Any]:
|
|
4323
4225
|
return dict(
|
|
4324
4226
|
**args,
|
|
4325
4227
|
dataset_id=self.id,
|
|
@@ -4329,12 +4231,12 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4329
4231
|
|
|
4330
4232
|
def insert(
|
|
4331
4233
|
self,
|
|
4332
|
-
input:
|
|
4333
|
-
expected:
|
|
4334
|
-
tags:
|
|
4335
|
-
metadata:
|
|
4336
|
-
id:
|
|
4337
|
-
output:
|
|
4234
|
+
input: Any | None = None,
|
|
4235
|
+
expected: Any | None = None,
|
|
4236
|
+
tags: Sequence[str] | None = None,
|
|
4237
|
+
metadata: dict[str, Any] | None = None,
|
|
4238
|
+
id: str | None = None,
|
|
4239
|
+
output: Any | None = None,
|
|
4338
4240
|
) -> str:
|
|
4339
4241
|
"""
|
|
4340
4242
|
Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
|
|
@@ -4373,10 +4275,10 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4373
4275
|
def update(
|
|
4374
4276
|
self,
|
|
4375
4277
|
id: str,
|
|
4376
|
-
input:
|
|
4377
|
-
expected:
|
|
4378
|
-
tags:
|
|
4379
|
-
metadata:
|
|
4278
|
+
input: Any | None = None,
|
|
4279
|
+
expected: Any | None = None,
|
|
4280
|
+
tags: Sequence[str] | None = None,
|
|
4281
|
+
metadata: dict[str, Any] | None = None,
|
|
4380
4282
|
) -> str:
|
|
4381
4283
|
"""
|
|
4382
4284
|
Update fields of a single record in the dataset. The updated fields will be batched and uploaded behind the scenes.
|
|
@@ -4420,8 +4322,7 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4420
4322
|
"_object_delete": True, # XXX potentially place this in the logging endpoint
|
|
4421
4323
|
},
|
|
4422
4324
|
)
|
|
4423
|
-
|
|
4424
|
-
partial_args = _deep_copy_event(partial_args)
|
|
4325
|
+
partial_args = bt_safe_deep_copy(partial_args)
|
|
4425
4326
|
|
|
4426
4327
|
def compute_args():
|
|
4427
4328
|
return dict(
|
|
@@ -4488,7 +4389,7 @@ class Dataset(ObjectFetcher[DatasetEvent]):
|
|
|
4488
4389
|
def render_message(render: Callable[[str], str], message: PromptMessage):
|
|
4489
4390
|
base = {k: v for (k, v) in message.as_dict().items() if v is not None}
|
|
4490
4391
|
# TODO: shouldn't load_prompt guarantee content is a PromptMessage?
|
|
4491
|
-
content = cast(Union[str,
|
|
4392
|
+
content = cast(Union[str, list[Union[TextPart, ImagePart]], dict[str, Any]], message.content)
|
|
4492
4393
|
if content is not None:
|
|
4493
4394
|
if isinstance(content, str):
|
|
4494
4395
|
base["content"] = render(content)
|
|
@@ -4552,7 +4453,7 @@ def render_message(render: Callable[[str], str], message: PromptMessage):
|
|
|
4552
4453
|
|
|
4553
4454
|
|
|
4554
4455
|
def _create_custom_render():
|
|
4555
|
-
def _get_key(key: str, scopes:
|
|
4456
|
+
def _get_key(key: str, scopes: list[dict[str, Any]], warn: bool) -> Any:
|
|
4556
4457
|
thing = chevron.renderer._get_key(key, scopes, warn) # type: ignore
|
|
4557
4458
|
if isinstance(thing, str):
|
|
4558
4459
|
return thing
|
|
@@ -4592,7 +4493,7 @@ def render_templated_object(obj: Any, args: Any) -> Any:
|
|
|
4592
4493
|
return obj
|
|
4593
4494
|
|
|
4594
4495
|
|
|
4595
|
-
def render_prompt_params(params:
|
|
4496
|
+
def render_prompt_params(params: dict[str, Any], args: Any) -> dict[str, Any]:
|
|
4596
4497
|
if not params:
|
|
4597
4498
|
return params
|
|
4598
4499
|
|
|
@@ -4617,7 +4518,7 @@ def render_prompt_params(params: Dict[str, Any], args: Any) -> Dict[str, Any]:
|
|
|
4617
4518
|
return {**params, "response_format": {**response_format, "json_schema": {**json_schema, "schema": parsed_schema}}}
|
|
4618
4519
|
|
|
4619
4520
|
|
|
4620
|
-
def render_mustache(template: str, data: Any, *, strict: bool = False, renderer:
|
|
4521
|
+
def render_mustache(template: str, data: Any, *, strict: bool = False, renderer: Callable[..., Any] | None = None):
|
|
4621
4522
|
if renderer is None:
|
|
4622
4523
|
renderer = chevron.render
|
|
4623
4524
|
|
|
@@ -4694,7 +4595,7 @@ class Prompt:
|
|
|
4694
4595
|
return self._lazy_metadata.get().slug
|
|
4695
4596
|
|
|
4696
4597
|
@property
|
|
4697
|
-
def prompt(self) ->
|
|
4598
|
+
def prompt(self) -> PromptBlockData | None:
|
|
4698
4599
|
return self._lazy_metadata.get().prompt_data.prompt
|
|
4699
4600
|
|
|
4700
4601
|
@property
|
|
@@ -4791,7 +4692,7 @@ class Prompt:
|
|
|
4791
4692
|
|
|
4792
4693
|
|
|
4793
4694
|
class Project:
|
|
4794
|
-
def __init__(self, name:
|
|
4695
|
+
def __init__(self, name: str | None = None, id: str | None = None):
|
|
4795
4696
|
self._name = name
|
|
4796
4697
|
self._id = id
|
|
4797
4698
|
self.init_lock = threading.RLock()
|
|
@@ -4831,9 +4732,9 @@ class Logger(Exportable):
|
|
|
4831
4732
|
self,
|
|
4832
4733
|
lazy_metadata: LazyValue[OrgProjectMetadata],
|
|
4833
4734
|
async_flush: bool = True,
|
|
4834
|
-
compute_metadata_args:
|
|
4835
|
-
link_args:
|
|
4836
|
-
state:
|
|
4735
|
+
compute_metadata_args: dict | None = None,
|
|
4736
|
+
link_args: dict | None = None,
|
|
4737
|
+
state: BraintrustState | None = None,
|
|
4837
4738
|
):
|
|
4838
4739
|
self._lazy_metadata = lazy_metadata
|
|
4839
4740
|
self.async_flush = async_flush
|
|
@@ -4873,15 +4774,15 @@ class Logger(Exportable):
|
|
|
4873
4774
|
|
|
4874
4775
|
def log(
|
|
4875
4776
|
self,
|
|
4876
|
-
input:
|
|
4877
|
-
output:
|
|
4878
|
-
expected:
|
|
4879
|
-
error:
|
|
4880
|
-
tags:
|
|
4881
|
-
scores:
|
|
4882
|
-
metadata:
|
|
4883
|
-
metrics:
|
|
4884
|
-
id:
|
|
4777
|
+
input: Any | None = None,
|
|
4778
|
+
output: Any | None = None,
|
|
4779
|
+
expected: Any | None = None,
|
|
4780
|
+
error: str | None = None,
|
|
4781
|
+
tags: Sequence[str] | None = None,
|
|
4782
|
+
scores: Mapping[str, int | float] | None = None,
|
|
4783
|
+
metadata: Mapping[str, Any] | None = None,
|
|
4784
|
+
metrics: Mapping[str, int | float] | None = None,
|
|
4785
|
+
id: str | None = None,
|
|
4885
4786
|
allow_concurrent_with_spans: bool = False,
|
|
4886
4787
|
) -> str:
|
|
4887
4788
|
"""
|
|
@@ -4926,11 +4827,11 @@ class Logger(Exportable):
|
|
|
4926
4827
|
def log_feedback(
|
|
4927
4828
|
self,
|
|
4928
4829
|
id: str,
|
|
4929
|
-
scores:
|
|
4930
|
-
expected:
|
|
4931
|
-
tags:
|
|
4932
|
-
comment:
|
|
4933
|
-
metadata:
|
|
4830
|
+
scores: Mapping[str, int | float] | None = None,
|
|
4831
|
+
expected: Any | None = None,
|
|
4832
|
+
tags: Sequence[str] | None = None,
|
|
4833
|
+
comment: str | None = None,
|
|
4834
|
+
metadata: Mapping[str, Any] | None = None,
|
|
4934
4835
|
source: Literal["external", "app", "api", None] = None,
|
|
4935
4836
|
) -> None:
|
|
4936
4837
|
"""
|
|
@@ -4958,15 +4859,15 @@ class Logger(Exportable):
|
|
|
4958
4859
|
|
|
4959
4860
|
def start_span(
|
|
4960
4861
|
self,
|
|
4961
|
-
name:
|
|
4962
|
-
type:
|
|
4963
|
-
span_attributes:
|
|
4964
|
-
start_time:
|
|
4965
|
-
set_current:
|
|
4966
|
-
parent:
|
|
4967
|
-
propagated_event:
|
|
4968
|
-
span_id:
|
|
4969
|
-
root_span_id:
|
|
4862
|
+
name: str | None = None,
|
|
4863
|
+
type: SpanTypeAttribute | None = None,
|
|
4864
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
4865
|
+
start_time: float | None = None,
|
|
4866
|
+
set_current: bool | None = None,
|
|
4867
|
+
parent: str | None = None,
|
|
4868
|
+
propagated_event: dict[str, Any] | None = None,
|
|
4869
|
+
span_id: str | None = None,
|
|
4870
|
+
root_span_id: str | None = None,
|
|
4970
4871
|
**event: Any,
|
|
4971
4872
|
) -> Span:
|
|
4972
4873
|
"""Create a new toplevel span underneath the logger. The name defaults to "root" and the span type to "task".
|
|
@@ -5004,15 +4905,15 @@ class Logger(Exportable):
|
|
|
5004
4905
|
|
|
5005
4906
|
def _start_span_impl(
|
|
5006
4907
|
self,
|
|
5007
|
-
name:
|
|
5008
|
-
type:
|
|
5009
|
-
span_attributes:
|
|
5010
|
-
start_time:
|
|
5011
|
-
set_current:
|
|
5012
|
-
parent:
|
|
5013
|
-
propagated_event:
|
|
5014
|
-
span_id:
|
|
5015
|
-
root_span_id:
|
|
4908
|
+
name: str | None = None,
|
|
4909
|
+
type: SpanTypeAttribute | None = None,
|
|
4910
|
+
span_attributes: SpanAttributes | Mapping[str, Any] | None = None,
|
|
4911
|
+
start_time: float | None = None,
|
|
4912
|
+
set_current: bool | None = None,
|
|
4913
|
+
parent: str | None = None,
|
|
4914
|
+
propagated_event: dict[str, Any] | None = None,
|
|
4915
|
+
span_id: str | None = None,
|
|
4916
|
+
root_span_id: str | None = None,
|
|
5016
4917
|
lookup_span_parent: bool = True,
|
|
5017
4918
|
**event: Any,
|
|
5018
4919
|
) -> Span:
|
|
@@ -5062,7 +4963,7 @@ class Logger(Exportable):
|
|
|
5062
4963
|
def __enter__(self) -> "Logger":
|
|
5063
4964
|
return self
|
|
5064
4965
|
|
|
5065
|
-
def _get_link_base_url(self) ->
|
|
4966
|
+
def _get_link_base_url(self) -> str | None:
|
|
5066
4967
|
"""Return the base of link urls (e.g. https://braintrust.dev/app/my-org-name/) if we have the info
|
|
5067
4968
|
otherwise return None.
|
|
5068
4969
|
"""
|
|
@@ -5098,11 +4999,11 @@ class ScoreSummary(SerializableDataClass):
|
|
|
5098
4999
|
score: float
|
|
5099
5000
|
"""Average score across all examples."""
|
|
5100
5001
|
|
|
5101
|
-
improvements:
|
|
5002
|
+
improvements: int | None
|
|
5102
5003
|
"""Number of improvements in the score."""
|
|
5103
|
-
regressions:
|
|
5004
|
+
regressions: int | None
|
|
5104
5005
|
"""Number of regressions in the score."""
|
|
5105
|
-
diff:
|
|
5006
|
+
diff: float | None = None
|
|
5106
5007
|
"""Difference in score between the current and reference experiment."""
|
|
5107
5008
|
|
|
5108
5009
|
def __str__(self):
|
|
@@ -5133,15 +5034,15 @@ class MetricSummary(SerializableDataClass):
|
|
|
5133
5034
|
# Used to help with formatting
|
|
5134
5035
|
_longest_metric_name: int
|
|
5135
5036
|
|
|
5136
|
-
metric:
|
|
5037
|
+
metric: float | int
|
|
5137
5038
|
"""Average metric across all examples."""
|
|
5138
5039
|
unit: str
|
|
5139
5040
|
"""Unit label for the metric."""
|
|
5140
|
-
improvements:
|
|
5041
|
+
improvements: int | None
|
|
5141
5042
|
"""Number of improvements in the metric."""
|
|
5142
|
-
regressions:
|
|
5043
|
+
regressions: int | None
|
|
5143
5044
|
"""Number of regressions in the metric."""
|
|
5144
|
-
diff:
|
|
5045
|
+
diff: float | None = None
|
|
5145
5046
|
"""Difference in metric between the current and reference experiment."""
|
|
5146
5047
|
|
|
5147
5048
|
def __str__(self):
|
|
@@ -5167,21 +5068,21 @@ class ExperimentSummary(SerializableDataClass):
|
|
|
5167
5068
|
|
|
5168
5069
|
project_name: str
|
|
5169
5070
|
"""Name of the project that the experiment belongs to."""
|
|
5170
|
-
project_id:
|
|
5071
|
+
project_id: str | None
|
|
5171
5072
|
"""ID of the project. May be `None` if the eval was run locally."""
|
|
5172
|
-
experiment_id:
|
|
5073
|
+
experiment_id: str | None
|
|
5173
5074
|
"""ID of the experiment. May be `None` if the eval was run locally."""
|
|
5174
5075
|
experiment_name: str
|
|
5175
5076
|
"""Name of the experiment."""
|
|
5176
|
-
project_url:
|
|
5077
|
+
project_url: str | None
|
|
5177
5078
|
"""URL to the project's page in the Braintrust app."""
|
|
5178
|
-
experiment_url:
|
|
5079
|
+
experiment_url: str | None
|
|
5179
5080
|
"""URL to the experiment's page in the Braintrust app."""
|
|
5180
|
-
comparison_experiment_name:
|
|
5081
|
+
comparison_experiment_name: str | None
|
|
5181
5082
|
"""The experiment scores are baselined against."""
|
|
5182
|
-
scores:
|
|
5083
|
+
scores: dict[str, ScoreSummary]
|
|
5183
5084
|
"""Summary of the experiment's scores."""
|
|
5184
|
-
metrics:
|
|
5085
|
+
metrics: dict[str, MetricSummary]
|
|
5185
5086
|
"""Summary of the experiment's metrics."""
|
|
5186
5087
|
|
|
5187
5088
|
def __str__(self):
|
|
@@ -5230,7 +5131,7 @@ class DatasetSummary(SerializableDataClass):
|
|
|
5230
5131
|
"""URL to the project's page in the Braintrust app."""
|
|
5231
5132
|
dataset_url: str
|
|
5232
5133
|
"""URL to the experiment's page in the Braintrust app."""
|
|
5233
|
-
data_summary:
|
|
5134
|
+
data_summary: DataSummary | None
|
|
5234
5135
|
"""Summary of the dataset's data."""
|
|
5235
5136
|
|
|
5236
5137
|
def __str__(self):
|
|
@@ -5245,7 +5146,8 @@ class DatasetSummary(SerializableDataClass):
|
|
|
5245
5146
|
|
|
5246
5147
|
|
|
5247
5148
|
class TracedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
|
|
5248
|
-
# Returns Any because Future
|
|
5149
|
+
# Returns Any because Future[T] generic typing was stabilized in Python 3.9,
|
|
5150
|
+
# but we maintain compatibility with older type checkers.
|
|
5249
5151
|
def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
|
|
5250
5152
|
# Capture all current context variables
|
|
5251
5153
|
context = contextvars.copy_context()
|
|
@@ -5257,7 +5159,7 @@ class TracedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
|
|
|
5257
5159
|
return super().submit(wrapped_fn, *args, **kwargs)
|
|
5258
5160
|
|
|
5259
5161
|
|
|
5260
|
-
def get_prompt_versions(project_id: str, prompt_id: str) ->
|
|
5162
|
+
def get_prompt_versions(project_id: str, prompt_id: str) -> list[str]:
|
|
5261
5163
|
"""
|
|
5262
5164
|
Get the versions for a specific prompt.
|
|
5263
5165
|
|
|
@@ -5317,13 +5219,13 @@ def get_prompt_versions(project_id: str, prompt_id: str) -> List[str]:
|
|
|
5317
5219
|
]
|
|
5318
5220
|
|
|
5319
5221
|
|
|
5320
|
-
def _get_app_url(app_url:
|
|
5222
|
+
def _get_app_url(app_url: str | None = None) -> str:
|
|
5321
5223
|
if app_url:
|
|
5322
5224
|
return app_url
|
|
5323
5225
|
return os.getenv("BRAINTRUST_APP_URL", DEFAULT_APP_URL)
|
|
5324
5226
|
|
|
5325
5227
|
|
|
5326
|
-
def _get_org_name(org_name:
|
|
5228
|
+
def _get_org_name(org_name: str | None = None) -> str | None:
|
|
5327
5229
|
if org_name:
|
|
5328
5230
|
return org_name
|
|
5329
5231
|
return os.getenv("BRAINTRUST_ORG_NAME")
|