braintrust 0.5.0__py3-none-any.whl → 0.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- braintrust/__init__.py +14 -0
- braintrust/_generated_types.py +56 -3
- braintrust/auto.py +179 -0
- braintrust/conftest.py +23 -4
- braintrust/db_fields.py +10 -0
- braintrust/framework.py +18 -5
- braintrust/generated_types.py +3 -1
- braintrust/logger.py +369 -134
- braintrust/merge_row_batch.py +49 -109
- braintrust/oai.py +51 -0
- braintrust/test_bt_json.py +0 -5
- braintrust/test_context.py +1264 -0
- braintrust/test_framework.py +37 -0
- braintrust/test_http.py +444 -0
- braintrust/test_logger.py +179 -5
- braintrust/test_merge_row_batch.py +160 -0
- braintrust/test_util.py +58 -1
- braintrust/util.py +20 -0
- braintrust/version.py +2 -2
- braintrust/wrappers/agno/__init__.py +2 -3
- braintrust/wrappers/anthropic.py +64 -0
- braintrust/wrappers/claude_agent_sdk/__init__.py +2 -3
- braintrust/wrappers/claude_agent_sdk/test_wrapper.py +9 -0
- braintrust/wrappers/dspy.py +52 -1
- braintrust/wrappers/google_genai/__init__.py +9 -6
- braintrust/wrappers/litellm.py +6 -43
- braintrust/wrappers/pydantic_ai.py +2 -3
- braintrust/wrappers/test_agno.py +9 -0
- braintrust/wrappers/test_anthropic.py +156 -0
- braintrust/wrappers/test_dspy.py +117 -0
- braintrust/wrappers/test_google_genai.py +9 -0
- braintrust/wrappers/test_litellm.py +57 -55
- braintrust/wrappers/test_openai.py +253 -1
- braintrust/wrappers/test_pydantic_ai_integration.py +9 -0
- braintrust/wrappers/test_utils.py +79 -0
- braintrust/wrappers/threads.py +114 -0
- {braintrust-0.5.0.dist-info → braintrust-0.5.3.dist-info}/METADATA +1 -1
- {braintrust-0.5.0.dist-info → braintrust-0.5.3.dist-info}/RECORD +41 -37
- {braintrust-0.5.0.dist-info → braintrust-0.5.3.dist-info}/WHEEL +1 -1
- braintrust/graph_util.py +0 -147
- {braintrust-0.5.0.dist-info → braintrust-0.5.3.dist-info}/entry_points.txt +0 -0
- {braintrust-0.5.0.dist-info → braintrust-0.5.3.dist-info}/top_level.txt +0 -0
|
@@ -6,7 +6,7 @@ import openai
|
|
|
6
6
|
import pytest
|
|
7
7
|
from braintrust import logger, wrap_openai
|
|
8
8
|
from braintrust.test_helpers import assert_dict_matches, init_test_logger
|
|
9
|
-
from braintrust.wrappers.test_utils import assert_metrics_are_valid
|
|
9
|
+
from braintrust.wrappers.test_utils import assert_metrics_are_valid, run_in_subprocess, verify_autoinstrument_script
|
|
10
10
|
from openai import AsyncOpenAI
|
|
11
11
|
from openai._types import NOT_GIVEN
|
|
12
12
|
from pydantic import BaseModel
|
|
@@ -1681,3 +1681,255 @@ def test_braintrust_tracing_processor_trace_metadata_logging(memory_logger):
|
|
|
1681
1681
|
spans = memory_logger.pop()
|
|
1682
1682
|
root_span = spans[0]
|
|
1683
1683
|
assert root_span["metadata"]["conversation_id"] == "test-12345", "Should log trace metadata"
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
class TestPatchOpenAI:
|
|
1687
|
+
"""Tests for patch_openai()."""
|
|
1688
|
+
|
|
1689
|
+
def test_patch_openai_sets_wrapped_flag(self):
|
|
1690
|
+
"""patch_openai() should set __braintrust_wrapped__ on openai module."""
|
|
1691
|
+
result = run_in_subprocess("""
|
|
1692
|
+
from braintrust.oai import patch_openai
|
|
1693
|
+
import openai
|
|
1694
|
+
|
|
1695
|
+
assert not hasattr(openai, "__braintrust_wrapped__")
|
|
1696
|
+
patch_openai()
|
|
1697
|
+
assert hasattr(openai, "__braintrust_wrapped__")
|
|
1698
|
+
print("SUCCESS")
|
|
1699
|
+
""")
|
|
1700
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1701
|
+
assert "SUCCESS" in result.stdout
|
|
1702
|
+
|
|
1703
|
+
def test_patch_openai_wraps_new_clients(self):
|
|
1704
|
+
"""After patch_openai(), new OpenAI() clients should be wrapped."""
|
|
1705
|
+
result = run_in_subprocess("""
|
|
1706
|
+
from braintrust.oai import patch_openai
|
|
1707
|
+
patch_openai()
|
|
1708
|
+
|
|
1709
|
+
import openai
|
|
1710
|
+
client = openai.OpenAI(api_key="test-key")
|
|
1711
|
+
|
|
1712
|
+
# Check that chat completions is wrapped (our wrapper adds tracing)
|
|
1713
|
+
# The wrapper replaces client.chat with a wrapped version
|
|
1714
|
+
chat_type = type(client.chat).__name__
|
|
1715
|
+
print(f"chat_type={chat_type}")
|
|
1716
|
+
print("SUCCESS")
|
|
1717
|
+
""")
|
|
1718
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1719
|
+
assert "SUCCESS" in result.stdout
|
|
1720
|
+
|
|
1721
|
+
def test_patch_openai_creates_spans(self):
|
|
1722
|
+
"""patch_openai() should create spans when making API calls."""
|
|
1723
|
+
result = run_in_subprocess("""
|
|
1724
|
+
from braintrust.oai import patch_openai
|
|
1725
|
+
from braintrust.test_helpers import init_test_logger
|
|
1726
|
+
from braintrust import logger
|
|
1727
|
+
|
|
1728
|
+
# Set up memory logger
|
|
1729
|
+
init_test_logger("test-auto")
|
|
1730
|
+
with logger._internal_with_memory_background_logger() as memory_logger:
|
|
1731
|
+
patch_openai()
|
|
1732
|
+
|
|
1733
|
+
import openai
|
|
1734
|
+
client = openai.OpenAI()
|
|
1735
|
+
|
|
1736
|
+
# Make a call within a span context
|
|
1737
|
+
import braintrust
|
|
1738
|
+
with braintrust.start_span(name="test") as span:
|
|
1739
|
+
try:
|
|
1740
|
+
# This will fail without API key, but span should still be created
|
|
1741
|
+
client.chat.completions.create(
|
|
1742
|
+
model="gpt-4o-mini",
|
|
1743
|
+
messages=[{"role": "user", "content": "hi"}],
|
|
1744
|
+
)
|
|
1745
|
+
except Exception:
|
|
1746
|
+
pass # Expected without API key
|
|
1747
|
+
|
|
1748
|
+
# Check that spans were logged
|
|
1749
|
+
spans = memory_logger.pop()
|
|
1750
|
+
# Should have at least the parent span
|
|
1751
|
+
assert len(spans) >= 1, f"Expected spans, got {spans}"
|
|
1752
|
+
print("SUCCESS")
|
|
1753
|
+
""")
|
|
1754
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1755
|
+
assert "SUCCESS" in result.stdout
|
|
1756
|
+
|
|
1757
|
+
def test_patch_openai_before_import(self):
|
|
1758
|
+
"""patch_openai() should work when called before importing openai."""
|
|
1759
|
+
result = run_in_subprocess("""
|
|
1760
|
+
from braintrust.oai import patch_openai
|
|
1761
|
+
|
|
1762
|
+
# Patch BEFORE importing openai
|
|
1763
|
+
patch_openai()
|
|
1764
|
+
|
|
1765
|
+
import openai
|
|
1766
|
+
assert hasattr(openai, "__braintrust_wrapped__")
|
|
1767
|
+
|
|
1768
|
+
client = openai.OpenAI(api_key="test-key")
|
|
1769
|
+
print("SUCCESS")
|
|
1770
|
+
""")
|
|
1771
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1772
|
+
assert "SUCCESS" in result.stdout
|
|
1773
|
+
|
|
1774
|
+
def test_patch_openai_after_import(self):
|
|
1775
|
+
"""patch_openai() should work when called after importing openai."""
|
|
1776
|
+
result = run_in_subprocess("""
|
|
1777
|
+
import openai
|
|
1778
|
+
from braintrust.oai import patch_openai
|
|
1779
|
+
|
|
1780
|
+
# Patch AFTER importing openai
|
|
1781
|
+
patch_openai()
|
|
1782
|
+
|
|
1783
|
+
assert hasattr(openai, "__braintrust_wrapped__")
|
|
1784
|
+
|
|
1785
|
+
client = openai.OpenAI(api_key="test-key")
|
|
1786
|
+
print("SUCCESS")
|
|
1787
|
+
""")
|
|
1788
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1789
|
+
assert "SUCCESS" in result.stdout
|
|
1790
|
+
|
|
1791
|
+
def test_patch_openai_idempotent(self):
|
|
1792
|
+
"""Multiple patch_openai() calls should be safe."""
|
|
1793
|
+
result = run_in_subprocess("""
|
|
1794
|
+
from braintrust.oai import patch_openai
|
|
1795
|
+
import openai
|
|
1796
|
+
|
|
1797
|
+
patch_openai()
|
|
1798
|
+
patch_openai() # Second call - should be no-op, not double-wrap
|
|
1799
|
+
|
|
1800
|
+
# Verify we can still create clients
|
|
1801
|
+
client = openai.OpenAI(api_key="test-key")
|
|
1802
|
+
assert hasattr(client, "chat")
|
|
1803
|
+
print("SUCCESS")
|
|
1804
|
+
""")
|
|
1805
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1806
|
+
assert "SUCCESS" in result.stdout
|
|
1807
|
+
|
|
1808
|
+
def test_patch_openai_chains_with_other_patches(self):
|
|
1809
|
+
"""patch_openai() should chain with other libraries that patch OpenAI."""
|
|
1810
|
+
result = run_in_subprocess("""
|
|
1811
|
+
import openai
|
|
1812
|
+
|
|
1813
|
+
# Simulate another library (like Datadog) patching OpenAI first
|
|
1814
|
+
other_library_init_called = []
|
|
1815
|
+
|
|
1816
|
+
class OtherLibraryOpenAI(openai.OpenAI):
|
|
1817
|
+
def __init__(self, *args, **kwargs):
|
|
1818
|
+
other_library_init_called.append(True)
|
|
1819
|
+
super().__init__(*args, **kwargs)
|
|
1820
|
+
|
|
1821
|
+
openai.OpenAI = OtherLibraryOpenAI
|
|
1822
|
+
|
|
1823
|
+
# Now apply our patch - should subclass OtherLibraryOpenAI
|
|
1824
|
+
from braintrust.oai import patch_openai
|
|
1825
|
+
patch_openai()
|
|
1826
|
+
|
|
1827
|
+
# Create a client - both patches should run
|
|
1828
|
+
client = openai.OpenAI(api_key="test-key")
|
|
1829
|
+
|
|
1830
|
+
# Verify other library's __init__ was called (chaining works)
|
|
1831
|
+
assert len(other_library_init_called) == 1, "Other library's patch should have run"
|
|
1832
|
+
|
|
1833
|
+
# Verify our patch was applied (client has wrapped chat)
|
|
1834
|
+
assert hasattr(client, "chat"), "Client should have chat attribute"
|
|
1835
|
+
|
|
1836
|
+
print("SUCCESS")
|
|
1837
|
+
""")
|
|
1838
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1839
|
+
assert "SUCCESS" in result.stdout
|
|
1840
|
+
|
|
1841
|
+
def test_patch_openai_chains_async_client(self):
|
|
1842
|
+
"""patch_openai() should chain with other libraries for AsyncOpenAI too."""
|
|
1843
|
+
result = run_in_subprocess("""
|
|
1844
|
+
import openai
|
|
1845
|
+
|
|
1846
|
+
# Simulate another library patching AsyncOpenAI first
|
|
1847
|
+
other_library_init_called = []
|
|
1848
|
+
|
|
1849
|
+
class OtherLibraryAsyncOpenAI(openai.AsyncOpenAI):
|
|
1850
|
+
def __init__(self, *args, **kwargs):
|
|
1851
|
+
other_library_init_called.append(True)
|
|
1852
|
+
super().__init__(*args, **kwargs)
|
|
1853
|
+
|
|
1854
|
+
openai.AsyncOpenAI = OtherLibraryAsyncOpenAI
|
|
1855
|
+
|
|
1856
|
+
# Now apply our patch
|
|
1857
|
+
from braintrust.oai import patch_openai
|
|
1858
|
+
patch_openai()
|
|
1859
|
+
|
|
1860
|
+
# Create an async client - both patches should run
|
|
1861
|
+
client = openai.AsyncOpenAI(api_key="test-key")
|
|
1862
|
+
|
|
1863
|
+
# Verify other library's __init__ was called
|
|
1864
|
+
assert len(other_library_init_called) == 1, "Other library's patch should have run"
|
|
1865
|
+
|
|
1866
|
+
# Verify our patch was applied
|
|
1867
|
+
assert hasattr(client, "chat"), "Client should have chat attribute"
|
|
1868
|
+
|
|
1869
|
+
print("SUCCESS")
|
|
1870
|
+
""")
|
|
1871
|
+
assert result.returncode == 0, f"Failed: {result.stderr}"
|
|
1872
|
+
assert "SUCCESS" in result.stdout
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
class TestPatchOpenAISpans:
|
|
1876
|
+
"""VCR-based tests verifying that patch_openai() produces spans."""
|
|
1877
|
+
|
|
1878
|
+
@pytest.mark.vcr
|
|
1879
|
+
def test_patch_openai_creates_spans(self, memory_logger):
|
|
1880
|
+
"""patch_openai() should create spans when making API calls."""
|
|
1881
|
+
from braintrust.oai import patch_openai
|
|
1882
|
+
|
|
1883
|
+
assert not memory_logger.pop()
|
|
1884
|
+
|
|
1885
|
+
patch_openai()
|
|
1886
|
+
client = openai.OpenAI()
|
|
1887
|
+
response = client.chat.completions.create(
|
|
1888
|
+
model="gpt-4o-mini",
|
|
1889
|
+
messages=[{"role": "user", "content": "Say hi"}],
|
|
1890
|
+
)
|
|
1891
|
+
assert response.choices[0].message.content
|
|
1892
|
+
|
|
1893
|
+
# Verify span was created
|
|
1894
|
+
spans = memory_logger.pop()
|
|
1895
|
+
assert len(spans) == 1
|
|
1896
|
+
span = spans[0]
|
|
1897
|
+
assert span["metadata"]["provider"] == "openai"
|
|
1898
|
+
assert "gpt-4o-mini" in span["metadata"]["model"]
|
|
1899
|
+
assert span["input"]
|
|
1900
|
+
|
|
1901
|
+
|
|
1902
|
+
class TestPatchOpenAIAsyncSpans:
|
|
1903
|
+
"""VCR-based tests verifying that patch_openai() produces spans for async clients."""
|
|
1904
|
+
|
|
1905
|
+
@pytest.mark.vcr
|
|
1906
|
+
@pytest.mark.asyncio
|
|
1907
|
+
async def test_patch_openai_async_creates_spans(self, memory_logger):
|
|
1908
|
+
"""patch_openai() should create spans for async API calls."""
|
|
1909
|
+
from braintrust.oai import patch_openai
|
|
1910
|
+
|
|
1911
|
+
assert not memory_logger.pop()
|
|
1912
|
+
|
|
1913
|
+
patch_openai()
|
|
1914
|
+
client = openai.AsyncOpenAI()
|
|
1915
|
+
response = await client.chat.completions.create(
|
|
1916
|
+
model="gpt-4o-mini",
|
|
1917
|
+
messages=[{"role": "user", "content": "Say hi async"}],
|
|
1918
|
+
)
|
|
1919
|
+
assert response.choices[0].message.content
|
|
1920
|
+
|
|
1921
|
+
# Verify span was created
|
|
1922
|
+
spans = memory_logger.pop()
|
|
1923
|
+
assert len(spans) == 1
|
|
1924
|
+
span = spans[0]
|
|
1925
|
+
assert span["metadata"]["provider"] == "openai"
|
|
1926
|
+
assert "gpt-4o-mini" in span["metadata"]["model"]
|
|
1927
|
+
assert span["input"]
|
|
1928
|
+
|
|
1929
|
+
|
|
1930
|
+
class TestAutoInstrumentOpenAI:
|
|
1931
|
+
"""Tests for auto_instrument() with OpenAI."""
|
|
1932
|
+
|
|
1933
|
+
def test_auto_instrument_openai(self):
|
|
1934
|
+
"""Test auto_instrument patches OpenAI, creates spans, and uninstrument works."""
|
|
1935
|
+
verify_autoinstrument_script("test_auto_openai.py")
|
|
@@ -9,6 +9,7 @@ import pytest
|
|
|
9
9
|
from braintrust import logger, setup_pydantic_ai, traced
|
|
10
10
|
from braintrust.span_types import SpanTypeAttribute
|
|
11
11
|
from braintrust.test_helpers import init_test_logger
|
|
12
|
+
from braintrust.wrappers.test_utils import verify_autoinstrument_script
|
|
12
13
|
from pydantic import BaseModel
|
|
13
14
|
from pydantic_ai import Agent, ModelSettings
|
|
14
15
|
from pydantic_ai.messages import ModelRequest, UserPromptPart
|
|
@@ -2572,3 +2573,11 @@ async def test_attachment_in_result_data(memory_logger):
|
|
|
2572
2573
|
copied = bt_safe_deep_copy(result_data)
|
|
2573
2574
|
assert copied["output_file"] is ext_attachment
|
|
2574
2575
|
assert copied["success"] is True
|
|
2576
|
+
|
|
2577
|
+
|
|
2578
|
+
class TestAutoInstrumentPydanticAI:
|
|
2579
|
+
"""Tests for auto_instrument() with Pydantic AI."""
|
|
2580
|
+
|
|
2581
|
+
def test_auto_instrument_pydantic_ai(self):
|
|
2582
|
+
"""Test auto_instrument patches Pydantic AI and creates spans."""
|
|
2583
|
+
verify_autoinstrument_script("test_auto_pydantic_ai.py")
|
|
@@ -1,3 +1,59 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import subprocess
|
|
3
|
+
import sys
|
|
4
|
+
import textwrap
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import vcr
|
|
9
|
+
from braintrust import logger
|
|
10
|
+
from braintrust.conftest import get_vcr_config
|
|
11
|
+
from braintrust.test_helpers import init_test_logger
|
|
12
|
+
|
|
13
|
+
# Source directory paths (resolved to handle installed vs source locations)
|
|
14
|
+
_SOURCE_DIR = Path(__file__).resolve().parent
|
|
15
|
+
AUTO_TEST_SCRIPTS_DIR = _SOURCE_DIR / "auto_test_scripts"
|
|
16
|
+
|
|
17
|
+
# Cassettes dir can be overridden via env var for subprocess tests
|
|
18
|
+
CASSETTES_DIR = Path(os.environ.get("BRAINTRUST_CASSETTES_DIR", _SOURCE_DIR / "cassettes"))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run_in_subprocess(
|
|
22
|
+
code: str, timeout: int = 30, env: dict[str, str] | None = None
|
|
23
|
+
) -> subprocess.CompletedProcess:
|
|
24
|
+
"""Run Python code in a fresh subprocess."""
|
|
25
|
+
run_env = os.environ.copy()
|
|
26
|
+
if env:
|
|
27
|
+
run_env.update(env)
|
|
28
|
+
return subprocess.run(
|
|
29
|
+
[sys.executable, "-c", textwrap.dedent(code)],
|
|
30
|
+
capture_output=True,
|
|
31
|
+
text=True,
|
|
32
|
+
timeout=timeout,
|
|
33
|
+
env=run_env,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def verify_autoinstrument_script(script_name: str, timeout: int = 30) -> subprocess.CompletedProcess:
|
|
38
|
+
"""Run a test script from the auto_test_scripts directory.
|
|
39
|
+
|
|
40
|
+
Raises AssertionError if the script exits with non-zero code.
|
|
41
|
+
"""
|
|
42
|
+
script_path = AUTO_TEST_SCRIPTS_DIR / script_name
|
|
43
|
+
# Pass cassettes dir to subprocess since it may use installed package
|
|
44
|
+
env = os.environ.copy()
|
|
45
|
+
env["BRAINTRUST_CASSETTES_DIR"] = str(_SOURCE_DIR / "cassettes")
|
|
46
|
+
result = subprocess.run(
|
|
47
|
+
[sys.executable, str(script_path)],
|
|
48
|
+
capture_output=True,
|
|
49
|
+
text=True,
|
|
50
|
+
timeout=timeout,
|
|
51
|
+
env=env,
|
|
52
|
+
)
|
|
53
|
+
assert result.returncode == 0, f"Script {script_name} failed:\n{result.stderr}"
|
|
54
|
+
return result
|
|
55
|
+
|
|
56
|
+
|
|
1
57
|
def assert_metrics_are_valid(metrics, start=None, end=None):
|
|
2
58
|
assert metrics
|
|
3
59
|
# assert 0 < metrics["time_to_first_token"]
|
|
@@ -10,3 +66,26 @@ def assert_metrics_are_valid(metrics, start=None, end=None):
|
|
|
10
66
|
assert start <= metrics["start"] <= metrics["end"] <= end
|
|
11
67
|
else:
|
|
12
68
|
assert metrics["start"] <= metrics["end"]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@contextmanager
|
|
72
|
+
def autoinstrument_test_context(cassette_name: str):
|
|
73
|
+
"""Context manager for auto_instrument tests.
|
|
74
|
+
|
|
75
|
+
Sets up VCR and memory_logger, yields memory_logger for direct use.
|
|
76
|
+
|
|
77
|
+
Usage:
|
|
78
|
+
with autoinstrument_test_context("test_auto_openai") as memory_logger:
|
|
79
|
+
# make API call
|
|
80
|
+
spans = memory_logger.pop()
|
|
81
|
+
"""
|
|
82
|
+
cassette_path = CASSETTES_DIR / f"{cassette_name}.yaml"
|
|
83
|
+
|
|
84
|
+
init_test_logger("test-auto-instrument")
|
|
85
|
+
|
|
86
|
+
with logger._internal_with_memory_background_logger() as memory_logger:
|
|
87
|
+
memory_logger.pop() # Clear any prior spans
|
|
88
|
+
|
|
89
|
+
my_vcr = vcr.VCR(**get_vcr_config())
|
|
90
|
+
with my_vcr.use_cassette(str(cassette_path)):
|
|
91
|
+
yield memory_logger
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import contextvars
|
|
2
|
+
import functools
|
|
3
|
+
import logging
|
|
4
|
+
import threading
|
|
5
|
+
from concurrent import futures
|
|
6
|
+
from typing import Any, TypeVar
|
|
7
|
+
|
|
8
|
+
from wrapt import wrap_function_wrapper # pyright: ignore[reportUnknownVariableType, reportMissingTypeStubs]
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
__all__ = ["setup_threads", "patch_thread", "patch_thread_pool_executor"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def setup_threads() -> bool:
|
|
16
|
+
"""
|
|
17
|
+
Setup automatic context propagation for threading.
|
|
18
|
+
|
|
19
|
+
This patches stdlib threading primitives to automatically
|
|
20
|
+
propagate Braintrust context across thread boundaries.
|
|
21
|
+
|
|
22
|
+
Enable via:
|
|
23
|
+
- BRAINTRUST_INSTRUMENT_THREADS=true env var (automatic)
|
|
24
|
+
- Call this function directly (manual)
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
bool: True if instrumentation was successful, False otherwise.
|
|
28
|
+
"""
|
|
29
|
+
try:
|
|
30
|
+
patch_thread(threading.Thread)
|
|
31
|
+
patch_thread_pool_executor(futures.ThreadPoolExecutor)
|
|
32
|
+
|
|
33
|
+
logger.debug("Braintrust thread instrumentation enabled")
|
|
34
|
+
return True
|
|
35
|
+
|
|
36
|
+
except Exception as e:
|
|
37
|
+
logger.warning(f"Failed to enable thread instrumentation: {e}")
|
|
38
|
+
return False
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
T = TypeVar("T", bound=type[threading.Thread])
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def patch_thread(thread_cls: T) -> T:
|
|
45
|
+
if __is_patched(thread_cls):
|
|
46
|
+
return thread_cls
|
|
47
|
+
|
|
48
|
+
def _wrap_thread_start(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
|
|
49
|
+
try:
|
|
50
|
+
instance._braintrust_context = contextvars.copy_context()
|
|
51
|
+
except Exception as e:
|
|
52
|
+
logger.debug(f"Failed to capture context in thread start: {e}")
|
|
53
|
+
return wrapped(*args, **kwargs)
|
|
54
|
+
|
|
55
|
+
wrap_function_wrapper(thread_cls, "start", _wrap_thread_start)
|
|
56
|
+
|
|
57
|
+
def _wrap_thread_run(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
|
|
58
|
+
try:
|
|
59
|
+
if hasattr(instance, "_braintrust_context"):
|
|
60
|
+
return instance._braintrust_context.run(wrapped, *args, **kwargs)
|
|
61
|
+
except Exception as e:
|
|
62
|
+
logger.debug(f"Failed to restore context in thread run: {e}")
|
|
63
|
+
return wrapped(*args, **kwargs)
|
|
64
|
+
|
|
65
|
+
wrap_function_wrapper(thread_cls, "run", _wrap_thread_run)
|
|
66
|
+
|
|
67
|
+
__mark_patched(thread_cls)
|
|
68
|
+
return thread_cls
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def __is_patched(obj: Any) -> bool:
|
|
72
|
+
"""Check if an object has already been patched."""
|
|
73
|
+
return getattr(obj, "_braintrust_patched", False)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def __mark_patched(obj: Any) -> None:
|
|
77
|
+
setattr(obj, "_braintrust_patched", True)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
P = TypeVar("P", bound=type[futures.ThreadPoolExecutor])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def patch_thread_pool_executor(executor_cls: P) -> P:
|
|
84
|
+
if __is_patched(executor_cls):
|
|
85
|
+
return executor_cls
|
|
86
|
+
|
|
87
|
+
def _wrap_executor_submit(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
|
|
88
|
+
try:
|
|
89
|
+
if not args:
|
|
90
|
+
return wrapped(*args, **kwargs)
|
|
91
|
+
|
|
92
|
+
func = args[0]
|
|
93
|
+
ctx = contextvars.copy_context()
|
|
94
|
+
|
|
95
|
+
@functools.wraps(func)
|
|
96
|
+
def context_wrapper(*func_args: Any, **func_kwargs: Any) -> Any:
|
|
97
|
+
try:
|
|
98
|
+
return ctx.run(func, *func_args, **func_kwargs)
|
|
99
|
+
except Exception as e:
|
|
100
|
+
# context.run() can fail if token is invalid
|
|
101
|
+
logger.debug(f"Failed to run in captured context: {e}")
|
|
102
|
+
return func(*func_args, **func_kwargs)
|
|
103
|
+
|
|
104
|
+
new_args = (context_wrapper,) + args[1:]
|
|
105
|
+
return wrapped(*new_args, **kwargs)
|
|
106
|
+
except Exception as e:
|
|
107
|
+
# Wrapping can fail - fall back to original
|
|
108
|
+
logger.debug(f"Failed to wrap executor submit: {e}")
|
|
109
|
+
return wrapped(*args, **kwargs)
|
|
110
|
+
|
|
111
|
+
wrap_function_wrapper(executor_cls, "submit", _wrap_executor_submit)
|
|
112
|
+
|
|
113
|
+
__mark_patched(executor_cls)
|
|
114
|
+
return executor_cls
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
braintrust/__init__.py,sha256=
|
|
2
|
-
braintrust/_generated_types.py,sha256=
|
|
1
|
+
braintrust/__init__.py,sha256=DU4gzkV0R3nsWxp2da7iQS1MM_T9xHkrKSQE6nAnZbE,2627
|
|
2
|
+
braintrust/_generated_types.py,sha256=1gNbTwsy7LEBdLW-uYqzQ2v78ToAFX0PSmeE_NOJC18,101153
|
|
3
3
|
braintrust/audit.py,sha256=3GQKzuTcFquYdrJtABM-k3xMlOIqgVkfG6UyeQ8_028,461
|
|
4
|
+
braintrust/auto.py,sha256=wf4Jb7hYoGS0Agpx-YjUYEW7wwwUpyAJDp-l3A-y6c0,4792
|
|
4
5
|
braintrust/aws.py,sha256=OBz_SRyopgpCDSNvETLypzGwTXk-bNLn-Eisevnjfwo,377
|
|
5
6
|
braintrust/bt_json.py,sha256=VNunedFUEfbvVEBuACHLRsjDr8lLD3_nFwB54MXtcbY,9385
|
|
6
|
-
braintrust/conftest.py,sha256=
|
|
7
|
+
braintrust/conftest.py,sha256=qcS1T4tqXN5Ocaol6gEN9xGOXBqvTYPXXiBJjHnHC6Y,3140
|
|
7
8
|
braintrust/context.py,sha256=bOo1Li29lUsi2DOOllIDar0oRuQNbkLNzJ7cYq5JTbo,4126
|
|
8
|
-
braintrust/db_fields.py,sha256=
|
|
9
|
-
braintrust/framework.py,sha256=
|
|
9
|
+
braintrust/db_fields.py,sha256=AX-K5t7KqO-xHHOfVRv8bn1ww7gZd3RNIFfANkZ2W0U,709
|
|
10
|
+
braintrust/framework.py,sha256=KoXFKCprfeEHq-AqjarENcmHHcTuvr4PScWtMbIQ6zg,62635
|
|
10
11
|
braintrust/framework2.py,sha256=o0igz4vXbmn0jHJPhDYvx14rFnI3ntV8H6VJfyJYRtM,16542
|
|
11
|
-
braintrust/generated_types.py,sha256=
|
|
12
|
+
braintrust/generated_types.py,sha256=MUpi5l9XIW0bGqnYqa9DE7XbxDYsBI82ANyTVHz9DVg,5047
|
|
12
13
|
braintrust/git_fields.py,sha256=au5ayyuvt7y_ojE9LC98ypTZd3RgFdjhRc8eFxcjnto,1434
|
|
13
14
|
braintrust/gitutil.py,sha256=RsW7cawJMgaAaTw6WeB1sShyfflkPb7yH2x7yuRv10c,5642
|
|
14
|
-
braintrust/graph_util.py,sha256=Z2Uy8RaOq5iMe5mShhQqRDDIpXVitE-biVxDiFB-0Ds,5545
|
|
15
15
|
braintrust/http_headers.py,sha256=9ZsDcsAKG04SGowsgchZktD6rG_oSTKWa8QyGUPA4xE,154
|
|
16
16
|
braintrust/id_gen.py,sha256=4UWLWRhksf76IkYi4cKACSaQ3yNgausrMRlhiurhy74,1590
|
|
17
|
-
braintrust/logger.py,sha256=
|
|
18
|
-
braintrust/merge_row_batch.py,sha256=
|
|
19
|
-
braintrust/oai.py,sha256=
|
|
17
|
+
braintrust/logger.py,sha256=ofx0_8ywVk_iyyqOH-6kCgRaYmm_sO7guVJ5hW2ISNg,217467
|
|
18
|
+
braintrust/merge_row_batch.py,sha256=mCutumLDOpH8ArP_4K5swP93mve7gmgfAnQdkdjUZ5E,6271
|
|
19
|
+
braintrust/oai.py,sha256=K7NLw7-3U7TJCyBVeoOwpy_UOmGlpaXXJcHZgW-rzHM,37746
|
|
20
20
|
braintrust/object.py,sha256=vYLyYWncsqLD00zffZUJwGTSkcJF9IIXmgIzrx3Np5c,632
|
|
21
21
|
braintrust/parameters.py,sha256=sQWfw18QXdPSnMHsF7aRrPmP7Zx6HEz9vaTUXWreudg,5911
|
|
22
22
|
braintrust/prompt.py,sha256=pLzhXoBp7ebxNraZADflR4YQMW5Ycjmt5ucK80P4_h0,1875
|
|
@@ -31,12 +31,15 @@ braintrust/span_identifier_v2.py,sha256=2dLc-Vz8iWLISmL_-ebCyWnY-ysA7sMnBsQtKqzM
|
|
|
31
31
|
braintrust/span_identifier_v3.py,sha256=RAvOK0lK0huH952kI5X1Q9TaAloD5to8jgTCuYMMw6o,10356
|
|
32
32
|
braintrust/span_identifier_v4.py,sha256=uFT-OdzySo4uCeAaJC3VqH55Oy433xZGBdK2hiEsm2w,10044
|
|
33
33
|
braintrust/span_types.py,sha256=cpTzCwUj4yBPbPLnzR23-VXIU2E9nl_dsVCSVMvtSkc,376
|
|
34
|
-
braintrust/test_bt_json.py,sha256=
|
|
35
|
-
braintrust/
|
|
34
|
+
braintrust/test_bt_json.py,sha256=pokqmFSQ3m8aB0XN1EEsS-zbv_N4KCYzad6VyOFtwPw,25188
|
|
35
|
+
braintrust/test_context.py,sha256=4wZOhwzGgAxt-CcN2izAxhgcY975oh_HUMkZXRwwTys,42754
|
|
36
|
+
braintrust/test_framework.py,sha256=rPjJYfVzRSbjf8e0irVfCnE3q5Rl1O_2Dy2dPyC9-lo,18674
|
|
36
37
|
braintrust/test_framework2.py,sha256=pSEEmBIyszAiYnpEVvDZgJqIe3lQ3T807LmIuBqV98w,7235
|
|
37
38
|
braintrust/test_helpers.py,sha256=VSelzjkR2IHyy5QD6sYB_79VIXq-wEDslDwyx9H11kI,13528
|
|
39
|
+
braintrust/test_http.py,sha256=RiW07psmoxO9ZB5ZUN1K1hIan7l4_F9YC2mW0iqSMr0,15751
|
|
38
40
|
braintrust/test_id_gen.py,sha256=Oqp8Rxd1_9ptpEJSu_xi6STIquZpx9n7Kh-bGUKMQH0,2489
|
|
39
|
-
braintrust/test_logger.py,sha256=
|
|
41
|
+
braintrust/test_logger.py,sha256=zINMwUn2hT1_ElZRkVJP95CkTwtLhjUKpR5zIP7tn5Y,112653
|
|
42
|
+
braintrust/test_merge_row_batch.py,sha256=cQ_Hiq6wlWIs091Tqexgd3TXWPb0BJm1N1t7cCQ7ZM4,4892
|
|
40
43
|
braintrust/test_otel.py,sha256=EyLOBMIZegvGyqqWl4vAlIazZfz1s4tWbgSAM68-2mM,31356
|
|
41
44
|
braintrust/test_queue.py,sha256=872weDhTfxN1Vu4-cV8pZZ9XNE8UkX4WNcvc2mPtSDI,8431
|
|
42
45
|
braintrust/test_score.py,sha256=PpfzNeaYhC4shMFqF85EczuMT0FCTr6l1rdEYWLe4g0,5857
|
|
@@ -44,11 +47,11 @@ braintrust/test_serializable_data_class.py,sha256=b04Ym64YtC6GJRGbKIN4J20RG1QN1F
|
|
|
44
47
|
braintrust/test_span_cache.py,sha256=HCCpedhJVb24jwcf2sbnP4Vj7t5p2AsMGUOSc91I6po,8705
|
|
45
48
|
braintrust/test_span_components.py,sha256=6w0oDnDLuVbHygNienujk4JDqtSRR5A49AI-S8Pa5hY,16197
|
|
46
49
|
braintrust/test_trace.py,sha256=u1V6wC-bgOhHofeg6q196CCurH3L-5diYjRK_hRQCII,8383
|
|
47
|
-
braintrust/test_util.py,sha256=
|
|
50
|
+
braintrust/test_util.py,sha256=SuSKTmvNyaR9Rbgf2TYCUWxJpZHoA2BMx6n4nQfV_pM,8221
|
|
48
51
|
braintrust/test_version.py,sha256=hk5JKjEFbNJ_ONc1VEkqHquflzre34RpFhCEYLTK8iA,1051
|
|
49
52
|
braintrust/trace.py,sha256=PHxfaHApGP_MPMIndZw7atIa2iwKPETUoG-TbF2dv6A,13767
|
|
50
|
-
braintrust/util.py,sha256
|
|
51
|
-
braintrust/version.py,sha256=
|
|
53
|
+
braintrust/util.py,sha256=S-qMBNsT36r_3pJ4LNKZ-vvHRlJwy8Wy7M7NAdfNOug,8919
|
|
54
|
+
braintrust/version.py,sha256=u7mu2r_lw0F22zAYs8OHbW11HrkdIhoAK-RbDs2uQdA,117
|
|
52
55
|
braintrust/xact_ids.py,sha256=bdyp88HjlyIkglgLSqYlCYscdSH6EWVyE14sR90Xl1s,658
|
|
53
56
|
braintrust/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
57
|
braintrust/cli/__main__.py,sha256=wCBKHGVmn3IT_yMXk5qfDwyI2SV2gf1tLr0NTxm9T8k,1519
|
|
@@ -92,37 +95,38 @@ braintrust/prompt_cache/test_lru_cache.py,sha256=4NNIXSfYBtOma7KYum74UdeM83eYmcX
|
|
|
92
95
|
braintrust/prompt_cache/test_prompt_cache.py,sha256=x17Ru9eaix8jt6yMhRgEljD2vVe7ieA8uKhk3bszgSM,8447
|
|
93
96
|
braintrust/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
94
97
|
braintrust/wrappers/_anthropic_utils.py,sha256=tOj_rv5NVRXpCij0F3NPxOZg0XGAAXo8Y0IYpNY5DHg,3078
|
|
95
|
-
braintrust/wrappers/anthropic.py,sha256=
|
|
96
|
-
braintrust/wrappers/dspy.py,sha256=
|
|
98
|
+
braintrust/wrappers/anthropic.py,sha256=jJDmdI0PnE4rNqtJvFaYd3huMnCWRJKxM1kNOH_J8xQ,13792
|
|
99
|
+
braintrust/wrappers/dspy.py,sha256=DGvDkSteOqhNhD5R3XgYcIKAI1-Ui2WYUbjqBl88C2Y,14996
|
|
97
100
|
braintrust/wrappers/langchain.py,sha256=0aY5LuVA7BPkgWA0N6CwPG9EaPqRmVVfEPaM1kN4XZY,5028
|
|
98
101
|
braintrust/wrappers/langsmith_wrapper.py,sha256=mWhBnY6KypOlSQw4NrCwzif0Oe6SHV6LYGGiwiz-EJQ,17956
|
|
99
|
-
braintrust/wrappers/litellm.py,sha256=
|
|
102
|
+
braintrust/wrappers/litellm.py,sha256=SxyDrji84OReBGiIf7aP51iIIF8ocD44QmmnYJwVCbo,25168
|
|
100
103
|
braintrust/wrappers/openai.py,sha256=SZuT4ouJd8FRNXxy9zM_OGb2HNL9XGsFnwkHCk3LDAM,10563
|
|
101
|
-
braintrust/wrappers/pydantic_ai.py,sha256=
|
|
102
|
-
braintrust/wrappers/test_agno.py,sha256=
|
|
103
|
-
braintrust/wrappers/test_anthropic.py,sha256=
|
|
104
|
-
braintrust/wrappers/test_dspy.py,sha256=
|
|
105
|
-
braintrust/wrappers/test_google_genai.py,sha256=
|
|
104
|
+
braintrust/wrappers/pydantic_ai.py,sha256=dukJCdcfkRc0ue8nPHpI5DogwNvL9kooGJEbri2W97w,48435
|
|
105
|
+
braintrust/wrappers/test_agno.py,sha256=ad5w6CEWzNwmzbF4NwSvHWhw1SpSjrt-X29pCmBRkhY,3772
|
|
106
|
+
braintrust/wrappers/test_anthropic.py,sha256=VkmlP3_FrZ12x5YWxcbx11A-TEZUghGQeQ4OuT3R4Oc,21409
|
|
107
|
+
braintrust/wrappers/test_dspy.py,sha256=0hlNi-JR3SJdYvFXAzD7AKuFJO0aXrbGVCV19OxIXdo,6382
|
|
108
|
+
braintrust/wrappers/test_google_genai.py,sha256=BKkPfLFk2JG1UNJ3mFKzv5OjioWqo7EQ0WSwZzvDltI,20563
|
|
106
109
|
braintrust/wrappers/test_langsmith_wrapper.py,sha256=wEbPNy4o7VVvcuHcsCJ-sy2EATvBxhUXTYFBQNkKCjs,10449
|
|
107
|
-
braintrust/wrappers/test_litellm.py,sha256=
|
|
110
|
+
braintrust/wrappers/test_litellm.py,sha256=MKukVH-C-Mwc0-cVJZhkLdXwiXupXHc8EiWYKq0X8V0,23620
|
|
108
111
|
braintrust/wrappers/test_oai_attachments.py,sha256=_EtNXjQxPgqXmj6UYMZn9GF4GDZf8m_1_TrwiEk7HWQ,11100
|
|
109
|
-
braintrust/wrappers/test_openai.py,sha256=
|
|
112
|
+
braintrust/wrappers/test_openai.py,sha256=pC-5zO2CbeG1wmQ5IvbCQ5vXic5sZgTCTW7347QwbwI,69474
|
|
110
113
|
braintrust/wrappers/test_openrouter.py,sha256=8HUfILPugOMqcvttpq76KQrynFb0xZpazvta7TTSF6A,3849
|
|
111
|
-
braintrust/wrappers/test_pydantic_ai_integration.py,sha256=
|
|
114
|
+
braintrust/wrappers/test_pydantic_ai_integration.py,sha256=xUSkiY5HUI-Z9G_etjyX7rGlOjBEdEY9CwVHyWM3xEE,104460
|
|
112
115
|
braintrust/wrappers/test_pydantic_ai_wrap_openai.py,sha256=OO5NrbothkMr4v2sZ-EZLH7-yLj3k6TfdLG4uzXAsQk,5090
|
|
113
|
-
braintrust/wrappers/test_utils.py,sha256=
|
|
114
|
-
braintrust/wrappers/
|
|
116
|
+
braintrust/wrappers/test_utils.py,sha256=wpLNRSCAZ42eG4srJrGFMAlfBSkhaCOcn2QwJXkRQ7s,3005
|
|
117
|
+
braintrust/wrappers/threads.py,sha256=rGQO6aFmFstrXA4xPEZCGTZJ1FC7CAS_4zF6ikszprc,3682
|
|
118
|
+
braintrust/wrappers/agno/__init__.py,sha256=soVMMmm_nmMcX9rWxfmKw9ur03XNKRcDmlf3G4GbR4g,2306
|
|
115
119
|
braintrust/wrappers/agno/agent.py,sha256=m3HCxQNotJviJswGYMxxsOnMilF-DNGqeFZFJa2Zhs4,6473
|
|
116
120
|
braintrust/wrappers/agno/function_call.py,sha256=MSsHLGoBk98xGlm-CYnjE5pYBi-IJfokLhjtbYgJpX0,2206
|
|
117
121
|
braintrust/wrappers/agno/model.py,sha256=VVFHEPlJ210P8BmSNQp8LchulHo62s4xvitvrR535yc,10714
|
|
118
122
|
braintrust/wrappers/agno/team.py,sha256=hYCnmeunLk47PEkoOrlpbxFERjK-D4evR07ghoiO5UU,6452
|
|
119
123
|
braintrust/wrappers/agno/utils.py,sha256=_b_s2WzAJ5HIkZ8Qbq5ZKctsdFDh5tX3gFuRiT2o6g0,15685
|
|
120
|
-
braintrust/wrappers/claude_agent_sdk/__init__.py,sha256=
|
|
124
|
+
braintrust/wrappers/claude_agent_sdk/__init__.py,sha256=4FUE59ii39jVfhMAfkOcU-TJ5rh2d3sX6fLzo2EcCJ8,4281
|
|
121
125
|
braintrust/wrappers/claude_agent_sdk/_wrapper.py,sha256=UNZBcgTu7X71zvJKy6e-QQFuz9j8kRS6Kd1VOh3OOXA,17755
|
|
122
|
-
braintrust/wrappers/claude_agent_sdk/test_wrapper.py,sha256=
|
|
123
|
-
braintrust/wrappers/google_genai/__init__.py,sha256=
|
|
124
|
-
braintrust-0.5.
|
|
125
|
-
braintrust-0.5.
|
|
126
|
-
braintrust-0.5.
|
|
127
|
-
braintrust-0.5.
|
|
128
|
-
braintrust-0.5.
|
|
126
|
+
braintrust/wrappers/claude_agent_sdk/test_wrapper.py,sha256=RmSzTfDC3tkepEbc9S_KYFITsjGVyllYE1fF9roSqBk,10779
|
|
127
|
+
braintrust/wrappers/google_genai/__init__.py,sha256=C7MzKUr17CEcIbP0kg9L9IjN-6suE3NQ-oSYAjXdR_g,15832
|
|
128
|
+
braintrust-0.5.3.dist-info/METADATA,sha256=snLiF8lw3wJkTkmgZnbthtfApgwnFb7eKFNARCWK0mM,3753
|
|
129
|
+
braintrust-0.5.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
130
|
+
braintrust-0.5.3.dist-info/entry_points.txt,sha256=Zpc0_09g5xm8as5jHqqFq7fhwO0xHSNct_TrEMONS7Q,60
|
|
131
|
+
braintrust-0.5.3.dist-info/top_level.txt,sha256=hw1-y-UFMf60RzAr8x_eM7SThbIuWfQsQIbVvqSF83A,11
|
|
132
|
+
braintrust-0.5.3.dist-info/RECORD,,
|