braintrust 0.3.10__py3-none-any.whl → 0.3.12__py3-none-any.whl

braintrust/devserver/auth.py

@@ -10,6 +10,7 @@ from ..logger import BraintrustState
 ORIGIN_HEADER = "origin"
 BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token"
 BRAINTRUST_ORG_NAME_HEADER = "x-bt-org-name"
+BRAINTRUST_PROJECT_ID_HEADER = "x-bt-project-id"
 
 
 @dataclass
@@ -17,6 +18,7 @@ class RequestContext:
     app_origin: Optional[str]
     token: Optional[str]
     org_name: Optional[str]
+    project_id: Optional[str]
     state: Optional[BraintrustState]
 
 
@@ -56,6 +58,7 @@ class AuthorizationMiddleware(BaseHTTPMiddleware):
                 app_origin=extract_allowed_origin(request.headers.get(ORIGIN_HEADER)),
                 token=None,
                 org_name=request.headers.get(BRAINTRUST_ORG_NAME_HEADER),
+                project_id=request.headers.get(BRAINTRUST_PROJECT_ID_HEADER),
                 state=None,
             )
 

braintrust/devserver/cors.py

@@ -18,6 +18,7 @@ ALLOWED_HEADERS = [
     "x-bt-auth-token",
     "x-bt-parent",
     "x-bt-org-name",
+    "x-bt-project-id",
     "x-bt-stream-fmt",
     "x-bt-use-cache",
     "x-stainless-os",

braintrust/devserver/server.py

@@ -196,7 +196,7 @@ async def run_eval(request: Request) -> Union[JSONResponse, StreamingResponse]:
                     "state": state,
                     "scores": evaluator.scores
                     + [
-                        make_scorer(state, score["name"], score["function_id"])
+                        make_scorer(state, score["name"], score["function_id"], ctx.project_id)
                         for score in eval_data.get("scores", [])
                     ],
                     "stream": stream_fn,
@@ -305,7 +305,7 @@ def snake_to_camel(snake_str: str) -> str:
     return components[0] + "".join(x.title() for x in components[1:]) if components else snake_str
 
 
-def make_scorer(state: BraintrustState, name: str, score: FunctionId) -> EvalScorer[Any, Any]:
+def make_scorer(state: BraintrustState, name: str, score: FunctionId, project_id: Optional[str] = None) -> EvalScorer[Any, Any]:
     def scorer_fn(input, output, expected, metadata):
         request = {
             **score,
@@ -315,7 +315,10 @@ def make_scorer(state: BraintrustState, name: str, score: FunctionId) -> EvalSco
             "mode": "auto",
             "strict": True,
         }
-        result = state.proxy_conn().post("function/invoke", json=request, headers={"Accept": "application/json"})
+        headers = {"Accept": "application/json"}
+        if project_id:
+            headers["x-bt-project-id"] = project_id
+        result = state.proxy_conn().post("function/invoke", json=request, headers=headers)
         result.raise_for_status()
         data = result.json()
         return data

braintrust/devserver/test_server_integration.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import Any
 
 import pytest
-
 from braintrust.framework import _evals
 from braintrust.test_helpers import has_devserver_installed
 
@@ -28,9 +27,8 @@ def client():
         pytest.skip("Devserver dependencies not installed (requires .[cli])")
 
     # Import CLI dependencies inside the fixture
-    from starlette.testclient import TestClient
-
     from braintrust.devserver.server import create_app
+    from starlette.testclient import TestClient
 
     # Use the real simple_eval.py example
     eval_file = Path(__file__).parent.parent.parent.parent / "examples" / "evals" / "simple_eval.py"
@@ -39,8 +37,8 @@ def client():
     _evals.clear()
 
     # Load the eval file to register evaluators (but don't run them)
-    spec = __import__('importlib.util').util.spec_from_file_location("simple_eval", str(eval_file))
-    module = __import__('importlib.util').util.module_from_spec(spec)
+    spec = __import__("importlib.util").util.spec_from_file_location("simple_eval", str(eval_file))
+    module = __import__("importlib.util").util.module_from_spec(spec)
 
     # Get evaluators from the module without executing Eval()
     # We need to parse the file and extract the Evaluator definition
@@ -134,6 +132,7 @@ def parse_sse_events(response_text: str) -> list[dict[str, Any]]:
     return events
 
 
+@pytest.mark.skip
 @pytest.mark.vcr
 def test_eval_sse_streaming(client, api_key, org_name):
     """

braintrust/oai.py

@@ -845,7 +845,7 @@ def _parse_metrics_from_usage(usage: Any) -> Dict[str, Any]:
 
 
 def _is_numeric(v):
-    return isinstance(v, (int, float, complex))
+    return isinstance(v, (int, float, complex)) and not isinstance(v, bool)
 
 
 def prettify_params(params: Dict[str, Any]) -> Dict[str, Any]:

braintrust/version.py

@@ -1,4 +1,4 @@
-VERSION = "0.3.10"
+VERSION = "0.3.12"
 
 # this will be templated during the build
-GIT_COMMIT = "160c352d87c121ef1ea43e49831453b9e8d4d0f8"
+GIT_COMMIT = "8c2b2995c2097f5699c101f4ab5e653ff40014a1"

braintrust/wrappers/test_openrouter.py

@@ -0,0 +1,144 @@
+"""
+Tests to ensure wrap_openai works correctly with OpenRouter.
+
+OpenRouter is a popular API gateway that provides access to multiple LLM providers
+through an OpenAI-compatible interface. This test validates that our wrapper handles
+OpenRouter-specific response fields correctly (e.g., boolean `is_byok` in usage).
+"""
+
+import os
+import time
+
+import pytest
+from braintrust import logger, wrap_openai
+from braintrust.test_helpers import init_test_logger
+from braintrust.wrappers.test_utils import assert_metrics_are_valid
+from openai import AsyncOpenAI, OpenAI
+
+PROJECT_NAME = "test-openrouter"
+TEST_MODEL = "openai/gpt-4o-mini"
+
+
+@pytest.fixture(scope="module")
+def vcr_config():
+    return {
+        "filter_headers": [
+            "authorization",
+        ]
+    }
+
+
+@pytest.fixture
+def memory_logger():
+    init_test_logger(PROJECT_NAME)
+    with logger._internal_with_memory_background_logger() as bgl:
+        yield bgl
+
+
+def _get_client():
+    return OpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.environ.get("OPENROUTER_API_KEY"),
+    )
+
+
+def _get_async_client():
+    return AsyncOpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.environ.get("OPENROUTER_API_KEY"),
+    )
+
+
+@pytest.mark.vcr
+def test_openrouter_chat_completion_sync(memory_logger):
+    assert not memory_logger.pop()
+
+    client = wrap_openai(_get_client())
+
+    start = time.time()
+    response = client.chat.completions.create(
+        model=TEST_MODEL,
+        messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}],
+        max_tokens=10,
+    )
+    end = time.time()
+
+    assert response
+    assert response.choices[0].message.content
+    assert "4" in response.choices[0].message.content
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+
+    metrics = span["metrics"]
+    assert_metrics_are_valid(metrics, start, end)
+
+    # Ensure no boolean values in metrics (the original bug with is_byok)
+    for key, value in metrics.items():
+        assert not isinstance(value, bool), f"Metric {key} should not be a boolean"
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_openrouter_chat_completion_async(memory_logger):
+    """Test that wrap_openai works with OpenRouter's async client."""
+    assert not memory_logger.pop()
+
+    client = wrap_openai(_get_async_client())
+
+    start = time.time()
+    response = await client.chat.completions.create(
+        model=TEST_MODEL,
+        messages=[{"role": "user", "content": "What is 3+3? Reply with just the number."}],
+        max_tokens=10,
+    )
+    end = time.time()
+
+    assert response
+    assert response.choices[0].message.content
+    assert "6" in response.choices[0].message.content
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+
+    metrics = span["metrics"]
+    assert_metrics_are_valid(metrics, start, end)
+
+    for key, value in metrics.items():
+        assert not isinstance(value, bool), f"Metric {key} should not be a boolean"
+
+
+@pytest.mark.vcr
+def test_openrouter_streaming_sync(memory_logger):
+    """Test that wrap_openai works with OpenRouter's streaming responses."""
+    assert not memory_logger.pop()
+
+    client = wrap_openai(_get_client())
+
+    start = time.time()
+    chunks = []
+    stream = client.chat.completions.create(
+        model=TEST_MODEL,
+        messages=[{"role": "user", "content": "What is 5+5? Reply with just the number."}],
+        max_tokens=10,
+        stream=True,
+    )
+    for chunk in stream:
+        chunks.append(chunk)
+    end = time.time()
+
+    assert chunks
+    content = "".join(c.choices[0].delta.content or "" for c in chunks if c.choices)
+    assert "10" in content
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+
+    metrics = span["metrics"]
+    assert_metrics_are_valid(metrics, start, end)
+
+    for key, value in metrics.items():
+        assert not isinstance(value, bool), f"Metric {key} should not be a boolean"

{braintrust-0.3.10.dist-info → braintrust-0.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braintrust
-Version: 0.3.10
+Version: 0.3.12
 Summary: SDK for integrating Braintrust
 Home-page: https://www.braintrust.dev
 Author: Braintrust

{braintrust-0.3.10.dist-info → braintrust-0.3.12.dist-info}/RECORD

@@ -16,7 +16,7 @@ braintrust/http_headers.py,sha256=9ZsDcsAKG04SGowsgchZktD6rG_oSTKWa8QyGUPA4xE,15
 braintrust/id_gen.py,sha256=PVkz-pS-9AzgmnAgpV-jgOFFo4hfl6e3IP9dVt6FouQ,1595
 braintrust/logger.py,sha256=qIElwbRqzT_3mrQ1XiS7razFxADjL6775eukwkiIhA0,209333
 braintrust/merge_row_batch.py,sha256=NX4jRE9uuFB3Z7btrarQp_di84_NGTjvzpJhksn82W8,9882
-braintrust/oai.py,sha256=8gUtISG8oIwgMRcFtlfDndzaF_md1Po_4_DFZcH_PkQ,33159
+braintrust/oai.py,sha256=DW_i6bXXDprwZ_uwcdIE25MUxx5y5DDdqr0ucdIWP9I,33187
 braintrust/object.py,sha256=vYLyYWncsqLD00zffZUJwGTSkcJF9IIXmgIzrx3Np5c,632
 braintrust/parameters.py,sha256=E1NJGJZ4Qjhp2h4Rghrh4qdE0FiaXLGPapS_iAk8es8,5946
 braintrust/prompt.py,sha256=c7UR-UeJ8Hf-DGyFRyUxSgXEOkZFhVd9-IhvyPhy1xI,1938
@@ -42,7 +42,7 @@ braintrust/test_span_components.py,sha256=UnF6ZL4k41XZ-CnfbjuqLeK4MZLtHTMdID3CMh
 braintrust/test_util.py,sha256=gyqe2JspRP7oXlp6ENztZe2fdRTOEMZMKpQi00y1DSc,4538
 braintrust/test_version.py,sha256=hk5JKjEFbNJ_ONc1VEkqHquflzre34RpFhCEYLTK8iA,1051
 braintrust/util.py,sha256=Ec6sRkQw5BckGrFjdA4YTyu_2BaKmHh4tWDwAi_ysOw,7227
-braintrust/version.py,sha256=SJg7EMz9LRoqG1nnPiMC44laB2sXCOxz83gnpK4Ee5g,118
+braintrust/version.py,sha256=i1ljewUfl4VA2SYv_e8CfC5C1RUiCNBL8mOBlRusJHk,118
 braintrust/xact_ids.py,sha256=bdyp88HjlyIkglgLSqYlCYscdSH6EWVyE14sR90Xl1s,658
 braintrust/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 braintrust/cli/__main__.py,sha256=wCBKHGVmn3IT_yMXk5qfDwyI2SV2gf1tLr0NTxm9T8k,1519
@@ -58,16 +58,16 @@ braintrust/contrib/__init__.py,sha256=Dh-0yGnjLhjK34QTV8ZRnjRIXbLu3q8C1h9GQsRKP6
 braintrust/contrib/temporal/__init__.py,sha256=kyV2FwryszPGb6sR5H3xR9FijFbJslDC3k-d7RNZufE,16519
 braintrust/contrib/temporal/test_temporal.py,sha256=UJxogj9z6qXgVCPQ_FiUSIfJXvz2LMsfLKMgrboyLw0,18491
 braintrust/devserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-braintrust/devserver/auth.py,sha256=rRHNe31dbCKt8jSRsIx0DjyRqi98PZ5LtJ-ScGu2TfE,2683
+braintrust/devserver/auth.py,sha256=ZCtfLI16u35elXx6vDHGJTcpEV8wJVVfJALuuVHZxW0,2840
 braintrust/devserver/cache.py,sha256=UPl_Co4D4WqWK8ly8lozkLBEk609qlXiF5DMrOWpTHw,1804
-braintrust/devserver/cors.py,sha256=7spdtvhXHEJ6FLIQQygwi1S7D52PPTY2RteG8sw7YQw,5568
+braintrust/devserver/cors.py,sha256=Tp0C3KuR0jEbLcjnBA0HyEgzesW14OybYI-xqxHfMzk,5591
 braintrust/devserver/dataset.py,sha256=93iWWvmuWQut2uG2AJD63Q8fPZGA0Nc9y_r6X_XJRFM,2315
 braintrust/devserver/eval_hooks.py,sha256=T5LWmAZjqa8kiXDKXYZlEsx_VhbIoiv8_X10s2v9C7M,1767
 braintrust/devserver/schemas.py,sha256=q9_I1DoaF5clLmMCkQvpB1DvFhy2GE1dZCumAc0S76M,9098
-braintrust/devserver/server.py,sha256=8lU0WNweoGM9sK8IiKQj4RtnNupKVlNEBZpbcXP07fc,12455
+braintrust/devserver/server.py,sha256=vmpLYR8OihAINpygkvPPNbg0t4M1OPeCb8rSco6QY9s,12606
 braintrust/devserver/test_cached_login.py,sha256=8RxzS3UuBeGx2L5j4mzXSQnJzF7pL0d4AhS3MNzyqi8,3741
 braintrust/devserver/test_lru_cache.py,sha256=5YYJ5uFj7k4Z4PQQ-UOV7bLP5zBYVo-5jV5_hpthtgM,4164
-braintrust/devserver/test_server_integration.py,sha256=Nq6OyEd5UT4VH4XHMRB8cJ2KUXzzHtDeWGGwr4WLsYA,6695
+braintrust/devserver/test_server_integration.py,sha256=ygIYDsTCSqwa_rzQDtdX0mjkXQVy7qyJjntdoCgtjKM,6711
 braintrust/functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 braintrust/functions/constants.py,sha256=g_EDiSrfCltHr5QaQMQzJ3qy3D29X-11LanDqlicqB0,23
 braintrust/functions/invoke.py,sha256=IGs_Zz3UbuKJqpg7O1sVcVIOGTD7QfcTXN2Ep83CqUY,8651
@@ -96,6 +96,7 @@ braintrust/wrappers/test_dspy.py,sha256=x5vXu-NoFFjWGIcvFvXICVsDasQEQzPuSlxgzbHJ
 braintrust/wrappers/test_google_genai.py,sha256=20e4DqLUZmcpVGircj7vN0IxsS5FVjvveqOu0JNZ5jo,17596
 braintrust/wrappers/test_litellm.py,sha256=sNId6qnbQibT66aB_lsxX_HlaPF6GoMFaawgnF1A2a4,21428
 braintrust/wrappers/test_openai.py,sha256=UqKtoIlZlqiLC6x7Rbv9zROgDdE5zvXZObbFMKpmifw,60237
+braintrust/wrappers/test_openrouter.py,sha256=9wIYuwObFkADH8h5pzHFB4C9TcL_HTeq1KlODYhSMLs,3987
 braintrust/wrappers/test_pydantic_ai.py,sha256=uqWbk1D8iX3q9qh_Q8PkopNUfPQCBCvOSa4KkpuIqqE,5057
 braintrust/wrappers/test_utils.py,sha256=Qz7LYG5V0DK2KuTJ_YLGpO_Zr_LJFfJgZX_Ps8tlM_c,505
 braintrust/wrappers/agno/__init__.py,sha256=V0dnEhu2U0FabyEmwDbWv5x6rP1rdI8z9JLAB_8XuXk,2416
@@ -108,8 +109,8 @@ braintrust/wrappers/claude_agent_sdk/__init__.py,sha256=CSXJWy-z2fHF7h4VJjLSnXJv
 braintrust/wrappers/claude_agent_sdk/_wrapper.py,sha256=uzElIOwwPmF_Y5fbWcKWEPC8HnSzW7byzpiuVKK0TXE,15613
 braintrust/wrappers/claude_agent_sdk/test_wrapper.py,sha256=0NmohdECudFvWtc-5PbANtTXzexkkwIJhGbujydDrT8,6826
 braintrust/wrappers/google_genai/__init__.py,sha256=PGFMuR3c4Gc3SUt24eP7z5AzdS2Dc1uF1d3QPCnLnuo,16018
-braintrust-0.3.10.dist-info/METADATA,sha256=hJlOuBcUfgMXQhHh3nPhixLRZjZpcpcu8VX7YCFXNBE,3131
-braintrust-0.3.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-braintrust-0.3.10.dist-info/entry_points.txt,sha256=Zpc0_09g5xm8as5jHqqFq7fhwO0xHSNct_TrEMONS7Q,60
-braintrust-0.3.10.dist-info/top_level.txt,sha256=hw1-y-UFMf60RzAr8x_eM7SThbIuWfQsQIbVvqSF83A,11
-braintrust-0.3.10.dist-info/RECORD,,
+braintrust-0.3.12.dist-info/METADATA,sha256=LMIW6iDtc0F1i6QsqMWoUcJn6RREF0H9XJ0n6mHjYb4,3131
+braintrust-0.3.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+braintrust-0.3.12.dist-info/entry_points.txt,sha256=Zpc0_09g5xm8as5jHqqFq7fhwO0xHSNct_TrEMONS7Q,60
+braintrust-0.3.12.dist-info/top_level.txt,sha256=hw1-y-UFMf60RzAr8x_eM7SThbIuWfQsQIbVvqSF83A,11
+braintrust-0.3.12.dist-info/RECORD,,