braintrust 0.3.11__tar.gz → 0.3.12__tar.gz

{braintrust-0.3.11 → braintrust-0.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braintrust
-Version: 0.3.11
+Version: 0.3.12
 Summary: SDK for integrating Braintrust
 Home-page: https://www.braintrust.dev
 Author: Braintrust

{braintrust-0.3.11 → braintrust-0.3.12}/src/braintrust/devserver/auth.py

@@ -10,6 +10,7 @@ from ..logger import BraintrustState
 ORIGIN_HEADER = "origin"
 BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token"
 BRAINTRUST_ORG_NAME_HEADER = "x-bt-org-name"
+BRAINTRUST_PROJECT_ID_HEADER = "x-bt-project-id"
 
 
 @dataclass
@@ -17,6 +18,7 @@ class RequestContext:
     app_origin: Optional[str]
     token: Optional[str]
     org_name: Optional[str]
+    project_id: Optional[str]
     state: Optional[BraintrustState]
 
 
@@ -56,6 +58,7 @@ class AuthorizationMiddleware(BaseHTTPMiddleware):
                 app_origin=extract_allowed_origin(request.headers.get(ORIGIN_HEADER)),
                 token=None,
                 org_name=request.headers.get(BRAINTRUST_ORG_NAME_HEADER),
+                project_id=request.headers.get(BRAINTRUST_PROJECT_ID_HEADER),
                 state=None,
             )
 

{braintrust-0.3.11 → braintrust-0.3.12}/src/braintrust/devserver/cors.py

@@ -18,6 +18,7 @@ ALLOWED_HEADERS = [
     "x-bt-auth-token",
     "x-bt-parent",
     "x-bt-org-name",
+    "x-bt-project-id",
     "x-bt-stream-fmt",
     "x-bt-use-cache",
     "x-stainless-os",

{braintrust-0.3.11 → braintrust-0.3.12}/src/braintrust/devserver/server.py

@@ -196,7 +196,7 @@ async def run_eval(request: Request) -> Union[JSONResponse, StreamingResponse]:
                     "state": state,
                     "scores": evaluator.scores
                     + [
-                        make_scorer(state, score["name"], score["function_id"])
+                        make_scorer(state, score["name"], score["function_id"], ctx.project_id)
                         for score in eval_data.get("scores", [])
                     ],
                     "stream": stream_fn,
@@ -305,7 +305,7 @@ def snake_to_camel(snake_str: str) -> str:
     return components[0] + "".join(x.title() for x in components[1:]) if components else snake_str
 
 
-def make_scorer(state: BraintrustState, name: str, score: FunctionId) -> EvalScorer[Any, Any]:
+def make_scorer(state: BraintrustState, name: str, score: FunctionId, project_id: Optional[str] = None) -> EvalScorer[Any, Any]:
     def scorer_fn(input, output, expected, metadata):
         request = {
             **score,
@@ -315,7 +315,10 @@ def make_scorer(state: BraintrustState, name: str, score: FunctionId) -> EvalSco
             "mode": "auto",
             "strict": True,
         }
-        result = state.proxy_conn().post("function/invoke", json=request, headers={"Accept": "application/json"})
+        headers = {"Accept": "application/json"}
+        if project_id:
+            headers["x-bt-project-id"] = project_id
+        result = state.proxy_conn().post("function/invoke", json=request, headers=headers)
         result.raise_for_status()
         data = result.json()
         return data

braintrust-0.3.12/src/braintrust/version.py

@@ -0,0 +1,4 @@
+VERSION = "0.3.12"
+
+# this will be templated during the build
+GIT_COMMIT = "8c2b2995c2097f5699c101f4ab5e653ff40014a1"

{braintrust-0.3.11 → braintrust-0.3.12}/src/braintrust/wrappers/test_openai.py

@@ -1690,35 +1690,3 @@ def test_braintrust_tracing_processor_trace_metadata_logging(memory_logger):
     spans = memory_logger.pop()
     root_span = spans[0]
     assert root_span["metadata"]["conversation_id"] == "test-12345", "Should log trace metadata"
-
-
-def test_parse_metrics_excludes_booleans():
-    """Test that boolean fields in usage objects are excluded from metrics.
-
-    Reproduces issue where OpenRouter returns is_byok (a boolean) in the usage
-    object, which caused API validation errors: "Expected number, received boolean".
-
-    In Python, bool is a subclass of int, so isinstance(True, int) returns True.
-    The fix ensures _is_numeric explicitly excludes booleans.
-    """
-    from braintrust.oai import _parse_metrics_from_usage
-
-    # Simulate OpenRouter's usage object with boolean field
-    usage = {
-        "completion_tokens": 11,
-        "prompt_tokens": 8,
-        "total_tokens": 19,
-        "cost": 0.000104,
-        "is_byok": False,  # This boolean should be filtered out
-    }
-
-    metrics = _parse_metrics_from_usage(usage)
-
-    # Numeric fields should be included
-    assert metrics["completion_tokens"] == 11
-    assert metrics["prompt_tokens"] == 8
-    assert metrics["tokens"] == 19  # total_tokens gets renamed
-    assert metrics["cost"] == 0.000104
-
-    # Boolean field should NOT be in metrics (this was the bug)
-    assert "is_byok" not in metrics

braintrust-0.3.12/src/braintrust/wrappers/test_openrouter.py

@@ -0,0 +1,144 @@
+"""
+Tests to ensure wrap_openai works correctly with OpenRouter.
+
+OpenRouter is a popular API gateway that provides access to multiple LLM providers
+through an OpenAI-compatible interface. This test validates that our wrapper handles
+OpenRouter-specific response fields correctly (e.g., boolean `is_byok` in usage).
+"""
+
+import os
+import time
+
+import pytest
+from braintrust import logger, wrap_openai
+from braintrust.test_helpers import init_test_logger
+from braintrust.wrappers.test_utils import assert_metrics_are_valid
+from openai import AsyncOpenAI, OpenAI
+
+PROJECT_NAME = "test-openrouter"
+TEST_MODEL = "openai/gpt-4o-mini"
+
+
+@pytest.fixture(scope="module")
+def vcr_config():
+    return {
+        "filter_headers": [
+            "authorization",
+        ]
+    }
+
+
+@pytest.fixture
+def memory_logger():
+    init_test_logger(PROJECT_NAME)
+    with logger._internal_with_memory_background_logger() as bgl:
+        yield bgl
+
+
+def _get_client():
+    return OpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.environ.get("OPENROUTER_API_KEY"),
+    )
+
+
+def _get_async_client():
+    return AsyncOpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.environ.get("OPENROUTER_API_KEY"),
+    )
+
+
+@pytest.mark.vcr
+def test_openrouter_chat_completion_sync(memory_logger):
+    assert not memory_logger.pop()
+
+    client = wrap_openai(_get_client())
+
+    start = time.time()
+    response = client.chat.completions.create(
+        model=TEST_MODEL,
+        messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}],
+        max_tokens=10,
+    )
+    end = time.time()
+
+    assert response
+    assert response.choices[0].message.content
+    assert "4" in response.choices[0].message.content
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+
+    metrics = span["metrics"]
+    assert_metrics_are_valid(metrics, start, end)
+
+    # Ensure no boolean values in metrics (the original bug with is_byok)
+    for key, value in metrics.items():
+        assert not isinstance(value, bool), f"Metric {key} should not be a boolean"
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_openrouter_chat_completion_async(memory_logger):
+    """Test that wrap_openai works with OpenRouter's async client."""
+    assert not memory_logger.pop()
+
+    client = wrap_openai(_get_async_client())
+
+    start = time.time()
+    response = await client.chat.completions.create(
+        model=TEST_MODEL,
+        messages=[{"role": "user", "content": "What is 3+3? Reply with just the number."}],
+        max_tokens=10,
+    )
+    end = time.time()
+
+    assert response
+    assert response.choices[0].message.content
+    assert "6" in response.choices[0].message.content
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+
+    metrics = span["metrics"]
+    assert_metrics_are_valid(metrics, start, end)
+
+    for key, value in metrics.items():
+        assert not isinstance(value, bool), f"Metric {key} should not be a boolean"
+
+
+@pytest.mark.vcr
+def test_openrouter_streaming_sync(memory_logger):
+    """Test that wrap_openai works with OpenRouter's streaming responses."""
+    assert not memory_logger.pop()
+
+    client = wrap_openai(_get_client())
+
+    start = time.time()
+    chunks = []
+    stream = client.chat.completions.create(
+        model=TEST_MODEL,
+        messages=[{"role": "user", "content": "What is 5+5? Reply with just the number."}],
+        max_tokens=10,
+        stream=True,
+    )
+    for chunk in stream:
+        chunks.append(chunk)
+    end = time.time()
+
+    assert chunks
+    content = "".join(c.choices[0].delta.content or "" for c in chunks if c.choices)
+    assert "10" in content
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+
+    metrics = span["metrics"]
+    assert_metrics_are_valid(metrics, start, end)
+
+    for key, value in metrics.items():
+        assert not isinstance(value, bool), f"Metric {key} should not be a boolean"

{braintrust-0.3.11 → braintrust-0.3.12}/src/braintrust.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braintrust
-Version: 0.3.11
+Version: 0.3.12
 Summary: SDK for integrating Braintrust
 Home-page: https://www.braintrust.dev
 Author: Braintrust

{braintrust-0.3.11 → braintrust-0.3.12}/src/braintrust.egg-info/SOURCES.txt

@@ -104,6 +104,7 @@ src/braintrust/wrappers/test_dspy.py
 src/braintrust/wrappers/test_google_genai.py
 src/braintrust/wrappers/test_litellm.py
 src/braintrust/wrappers/test_openai.py
+src/braintrust/wrappers/test_openrouter.py
 src/braintrust/wrappers/test_pydantic_ai.py
 src/braintrust/wrappers/test_utils.py
 src/braintrust/wrappers/agno/__init__.py

braintrust-0.3.11/src/braintrust/version.py

@@ -1,4 +0,0 @@
-VERSION = "0.3.11"
-
-# this will be templated during the build
-GIT_COMMIT = "7cffe54f8b960931055ae20f5983b4ea34515eff"