braintrust 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

braintrust/test_framework.py

@@ -241,6 +241,62 @@ async def test_hooks_trial_index_multiple_inputs():
     assert sorted(input_2_trials) == [0, 1]
 
 
+@pytest.mark.asyncio
+async def test_scorer_spans_have_purpose_attribute(with_memory_logger, with_simulate_login):
+    """Test that scorer spans have span_attributes.purpose='scorer' and propagate to subspans."""
+    # Define test data
+    data = [
+        EvalCase(input="hello", expected="hello"),
+    ]
+
+    def simple_task(input_value):
+        return input_value
+
+    def purpose_scorer(input_value, output, expected):
+        return 1.0 if output == expected else 0.0
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-scorer-purpose",
+        data=data,
+        task=simple_task,
+        scores=[purpose_scorer],
+        experiment_name="test-scorer-purpose",
+        metadata=None,
+    )
+
+    # Create experiment so spans get logged
+    exp = init_test_exp("test-scorer-purpose", "test-project")
+
+    # Run evaluator
+    result = await run_evaluator(experiment=exp, evaluator=evaluator, position=None, filters=[])
+
+    assert len(result.results) == 1
+    assert result.results[0].scores.get("purpose_scorer") == 1.0
+
+    # Check the logged spans
+    logs = with_memory_logger.pop()
+
+    # Find the scorer span (has type="score")
+    scorer_spans = [log for log in logs if log.get("span_attributes", {}).get("type") == "score"]
+    assert len(scorer_spans) == 1, f"Expected 1 scorer span, found {len(scorer_spans)}"
+
+    scorer_span = scorer_spans[0]
+
+    # Verify the scorer span has purpose='scorer'
+    assert scorer_span["span_attributes"].get("purpose") == "scorer", (
+        f"Scorer span should have purpose='scorer', got: {scorer_span['span_attributes']}"
+    )
+
+    # Verify that non-scorer spans (task, eval) do NOT have purpose='scorer'
+    non_scorer_spans = [log for log in logs if log.get("span_attributes", {}).get("type") != "score"]
+    assert len(non_scorer_spans) > 0, "Expected at least one non-scorer span"
+    for span in non_scorer_spans:
+        assert span.get("span_attributes", {}).get("purpose") != "scorer", (
+            f"Non-scorer span should NOT have purpose='scorer', got: {span['span_attributes']}"
+        )
+
+
 @pytest.fixture
 def simple_scorer():
     def simple_scorer_function(input, output, expected):

braintrust/test_logger.py

@@ -1,3 +1,5 @@
+# pyright: reportUnknownVariableType=false
+# pyright: reportPrivateUsage=false
 import asyncio
 import json
 import logging
@@ -20,7 +22,12 @@ from braintrust import (
     logger,
 )
 from braintrust.id_gen import OTELIDGenerator, get_id_generator
-from braintrust.logger import _deep_copy_event, _extract_attachments, parent_context, render_message, render_mustache
+from braintrust.logger import (
+    _extract_attachments,
+    parent_context,
+    render_message,
+    render_mustache,
+)
 from braintrust.prompt import PromptChatBlock, PromptData, PromptMessage, PromptSchema
 from braintrust.test_helpers import (
     assert_dict_matches,
@@ -170,113 +177,7 @@ class TestLogger(TestCase):
             },
         )
 
-    def test_deep_copy_event_basic(self):
-        original = {
-            "input": {"foo": "bar", "null": None, "empty": {}},
-            "output": [1, 2, "3", None, {}],
-        }
-        copy = _deep_copy_event(original)
-        self.assertEqual(copy, original)
-        self.assertIsNot(copy, original)
-        self.assertIsNot(copy["input"], original["input"])
-        self.assertIsNot(copy["output"], original["output"])
-
-    def test_deep_copy_event_with_attachments(self):
-        attachment1 = Attachment(
-            data=b"data",
-            filename="filename",
-            content_type="text/plain",
-        )
-        attachment2 = Attachment(
-            data=b"data2",
-            filename="filename2",
-            content_type="text/plain",
-        )
-        attachment3 = ExternalAttachment(
-            url="s3://bucket/path/to/key.pdf",
-            filename="filename3",
-            content_type="application/pdf",
-        )
-        date = "2024-10-23T05:02:48.796Z"
-
-        original = {
-            "input": "Testing",
-            "output": {
-                "span": "<span>",
-                "myIllegalObjects": ["<experiment>", "<dataset>", "<logger>"],
-                "myOtherWeirdObjects": [None, date, None, None],
-                "attachment": attachment1,
-                "another_attachment": attachment3,
-                "attachmentList": [attachment1, attachment2, "string", attachment3],
-                "nestedAttachment": {
-                    "attachment": attachment2,
-                    "another_attachment": attachment3,
-                },
-                "fake": {
-                    "_bt_internal_saved_attachment": "not a number",
-                },
-            },
-        }
 
-        copy = _deep_copy_event(original)
-
-        self.assertEqual(
-            copy,
-            {
-                "input": "Testing",
-                "output": {
-                    "span": "<span>",
-                    "myIllegalObjects": ["<experiment>", "<dataset>", "<logger>"],
-                    "myOtherWeirdObjects": [None, date, None, None],
-                    "attachment": attachment1,
-                    "another_attachment": attachment3,
-                    "attachmentList": [attachment1, attachment2, "string", attachment3],
-                    "nestedAttachment": {
-                        "attachment": attachment2,
-                        "another_attachment": attachment3,
-                    },
-                    "fake": {
-                        "_bt_internal_saved_attachment": "not a number",
-                    },
-                },
-            },
-        )
-
-        self.assertIsNot(copy, original)
-
-        self.assertIs(copy["output"]["attachment"], attachment1)
-        self.assertIs(copy["output"]["another_attachment"], attachment3)
-        self.assertIs(copy["output"]["nestedAttachment"]["attachment"], attachment2)
-        self.assertIs(copy["output"]["nestedAttachment"]["another_attachment"], attachment3)
-        self.assertIs(copy["output"]["attachmentList"][0], attachment1)
-        self.assertIs(copy["output"]["attachmentList"][1], attachment2)
-        self.assertIs(copy["output"]["attachmentList"][3], attachment3)
-
-    def test_check_json_serializable_catches_circular_references(self):
-        """Test that _check_json_serializable properly handles circular references.
-
-        After fix, _check_json_serializable should catch ValueError from circular
-        references and convert them to a more appropriate exception or handle them.
-        """
-        from braintrust.logger import _check_json_serializable
-
-        # Create data with circular reference
-        data = {"a": "b"}
-        data["self"] = data
-
-        # Should either succeed (by handling circular refs) or raise a clear exception
-        # The error message should indicate the data is not serializable
-        try:
-            result = _check_json_serializable(data)
-            # If it succeeds, it should return a serialized string
-            self.assertIsInstance(result, str)
-        except Exception as e:
-            # If it raises an exception, it should mention serialization issue
-            error_msg = str(e).lower()
-            self.assertTrue(
-                "json-serializable" in error_msg or "circular" in error_msg,
-                f"Expected error message to mention serialization issue, got: {e}",
-            )
 
     def test_prompt_build_with_structured_output_templating(self):
         self.maxDiff = None
@@ -756,6 +657,25 @@ def test_span_log_handles_infinity_gracefully(with_memory_logger):
     assert logs[0]["output"]["neg"] == "-Infinity"
 
 
+def test_span_log_with_binary_data(with_memory_logger):
+    """Test how span.log() currently handles binary data."""
+    logger = init_test_logger(__name__)
+
+    with logger.start_span(name="test_span") as span:
+        span.log(
+            input={"file": "image.png"},
+            output={"embedding": b"\x00\x01\x02\x03" * 100},
+        )
+
+    logs = with_memory_logger.pop()
+    assert len(logs) == 1
+    # Document actual behavior - binary data goes through deep_copy_and_sanitize_dict
+    # which uses bt_dumps/bt_loads roundtrip
+    assert logs[0]["input"]["file"] == "image.png"
+    # The embedding should be present (converted to some serializable form)
+    assert "embedding" in logs[0]["output"]
+
+
 def test_span_log_handles_unstringifiable_object_gracefully(with_memory_logger):
     """Test that span.log() should handle objects with bad __str__ gracefully without raising.
 
@@ -2746,3 +2666,187 @@ class TestDatasetInternalBtql(TestCase):
 
         # Verify that the custom batch_size is used
         self.assertEqual(query_json["limit"], custom_batch_size)
+
+
+def test_attachment_identity_preserved_through_bt_safe_deep_copy():
+    """Test that attachment object identity is preserved through bt_safe_deep_copy."""
+    from braintrust.bt_json import bt_safe_deep_copy
+
+    attachment = Attachment(data=b"data", filename="file.txt", content_type="text/plain")
+    original_id = id(attachment)
+
+    # Simulate what happens in Span.log
+    partial_record = {"input": {"file": attachment}}
+    copied = bt_safe_deep_copy(partial_record)
+
+    # Verify identity preserved
+    assert copied["input"]["file"] is attachment
+    assert id(copied["input"]["file"]) == original_id
+
+
+def test_extract_attachments_collects_and_replaces():
+    """Test that _extract_attachments properly collects attachments and replaces them with references."""
+    from braintrust.logger import _extract_attachments
+
+    attachment1 = Attachment(data=b"data1", filename="file1.txt", content_type="text/plain")
+    attachment2 = Attachment(data=b"data2", filename="file2.txt", content_type="text/plain")
+    ext_attachment = ExternalAttachment(url="s3://bucket/key", filename="file3.pdf", content_type="application/pdf")
+
+    event = {
+        "input": {"file": attachment1},
+        "output": {"file": attachment2},
+        "metadata": {"files": [attachment1, ext_attachment]}
+    }
+
+    attachments = []
+    _extract_attachments(event, attachments)
+
+    # Should have collected all 4 attachment instances (attachment1 appears twice)
+    assert len(attachments) == 4
+    assert attachments[0] is attachment1
+    assert attachments[1] is attachment2
+    assert attachments[2] is attachment1  # Same instance collected again
+    assert attachments[3] is ext_attachment
+
+    # Event should have been modified to contain references
+    assert event["input"]["file"] == attachment1.reference
+    assert event["output"]["file"] == attachment2.reference
+    assert event["metadata"]["files"][0] == attachment1.reference
+    assert event["metadata"]["files"][1] == ext_attachment.reference
+
+
+def test_extract_attachments_preserves_identity():
+    """Test that the same attachment instance is collected multiple times when it appears in different places."""
+    from braintrust.logger import _extract_attachments
+
+    attachment = Attachment(data=b"data", filename="file.txt", content_type="text/plain")
+    original_id = id(attachment)
+
+    event = {
+        "input": attachment,
+        "output": attachment,  # Same instance
+        "metadata": {"file": attachment}  # Same instance again
+    }
+
+    attachments = []
+    _extract_attachments(event, attachments)
+
+    # Should collect the same instance 3 times
+    assert len(attachments) == 3
+    assert all(att is attachment for att in attachments)
+    assert all(id(att) == original_id for att in attachments)
+
+
+def test_attachment_upload_tracked_on_flush(with_memory_logger, with_simulate_login):
+    """Test that attachment upload is tracked when attachments are logged and flushed."""
+    attachment = Attachment(data=b"test data", filename="test.txt", content_type="text/plain")
+
+    logger = init_test_logger(__name__)
+    span = logger.start_span(name="test_span")
+    span.log(input={"file": attachment})
+    span.end()
+
+    # No upload attempts yet
+    assert len(with_memory_logger.upload_attempts) == 0
+
+    # Flush should track upload attempt
+    logger.flush()
+
+    # Now upload should be tracked
+    assert len(with_memory_logger.upload_attempts) == 1
+    assert with_memory_logger.upload_attempts[0] is attachment
+
+
+def test_multiple_attachments_upload_tracked(with_memory_logger, with_simulate_login):
+    """Test that upload is tracked for multiple attachments."""
+    attachment1 = Attachment(data=b"data1", filename="file1.txt", content_type="text/plain")
+    attachment2 = Attachment(data=b"data2", filename="file2.txt", content_type="text/plain")
+
+    logger = init_test_logger(__name__)
+    span = logger.start_span(name="test_span")
+    span.log(
+        input={"file1": attachment1},
+        output={"file2": attachment2}
+    )
+    span.end()
+    logger.flush()
+
+    # Both attachments should be tracked
+    assert len(with_memory_logger.upload_attempts) == 2
+    assert attachment1 in with_memory_logger.upload_attempts
+    assert attachment2 in with_memory_logger.upload_attempts
+
+
+def test_same_attachment_logged_twice_tracked_twice(with_memory_logger, with_simulate_login):
+    """Test that same attachment logged twice appears twice in upload attempts."""
+    attachment = Attachment(data=b"data", filename="file.txt", content_type="text/plain")
+
+    logger = init_test_logger(__name__)
+    span = logger.start_span(name="test_span")
+    span.log(input={"file": attachment})
+    span.log(metadata={"same_file": attachment})
+    span.end()
+    logger.flush()
+
+    # Same attachment should be tracked twice (once for each log call)
+    assert len(with_memory_logger.upload_attempts) == 2
+    assert with_memory_logger.upload_attempts[0] is attachment
+    assert with_memory_logger.upload_attempts[1] is attachment
+
+
+def test_external_attachment_upload_tracked(with_memory_logger, with_simulate_login):
+    """Test that ExternalAttachment upload is also tracked."""
+    ext_attachment = ExternalAttachment(
+        url="s3://bucket/key.pdf",
+        filename="external.pdf",
+        content_type="application/pdf"
+    )
+
+    logger = init_test_logger(__name__)
+    span = logger.start_span(name="test_span")
+    span.log(input={"file": ext_attachment})
+    span.end()
+    logger.flush()
+
+    # ExternalAttachment should be tracked
+    assert len(with_memory_logger.upload_attempts) == 1
+    assert with_memory_logger.upload_attempts[0] is ext_attachment
+
+
+def test_json_attachment_upload_tracked(with_memory_logger, with_simulate_login):
+    """Test that JSONAttachment upload is tracked."""
+    data = {"key": "value", "nested": {"array": [1, 2, 3]}}
+    json_attachment = JSONAttachment(data, filename="data.json")
+
+    logger = init_test_logger(__name__)
+    span = logger.start_span(name="test_span")
+    span.log(output={"data": json_attachment})
+    span.end()
+    logger.flush()
+
+    # JSONAttachment should be tracked
+    assert len(with_memory_logger.upload_attempts) == 1
+    assert with_memory_logger.upload_attempts[0] is json_attachment
+
+
+def test_multiple_attachment_types_tracked(with_memory_logger, with_simulate_login):
+    """Test that different attachment types are all tracked."""
+    attachment = Attachment(data=b"data", filename="file.txt", content_type="text/plain")
+    json_attachment = JSONAttachment({"key": "value"}, filename="data.json")
+    ext_attachment = ExternalAttachment(url="s3://bucket/key", filename="file.pdf", content_type="application/pdf")
+
+    logger = init_test_logger(__name__)
+    span = logger.start_span(name="test_span")
+    span.log(
+        input=attachment,
+        output=json_attachment,
+        metadata={"file": ext_attachment}
+    )
+    span.end()
+    logger.flush()
+
+    # All three types should be tracked
+    assert len(with_memory_logger.upload_attempts) == 3
+    assert attachment in with_memory_logger.upload_attempts
+    assert json_attachment in with_memory_logger.upload_attempts
+    assert ext_attachment in with_memory_logger.upload_attempts

braintrust/version.py

@@ -1,4 +1,4 @@
-VERSION = "0.4.0"
+VERSION = "0.4.1"
 
 # this will be templated during the build
-GIT_COMMIT = "8ab13f3f48af6a4d3c0b053e4bbabfd4f24f23ec"
+GIT_COMMIT = "d9c624ea93ca6bf62c2412abce1b3a2ef1a2be67"

braintrust/wrappers/google_genai/__init__.py

@@ -3,6 +3,7 @@ import time
 from collections.abc import Iterable
 from typing import Any
 
+from braintrust.bt_json import bt_safe_deep_copy
 from braintrust.logger import NOOP_SPAN, Attachment, current_span, init_logger, start_span
 from braintrust.span_types import SpanTypeAttribute
 from wrapt import wrap_function_wrapper
@@ -149,7 +150,7 @@ def wrap_async_models(AsyncModels: Any):
 
 
 def _serialize_input(api_client: Any, input: dict[str, Any]):
-    config = _try_dict(input.get("config"))
+    config = bt_safe_deep_copy(input.get("config"))
 
     if config is not None:
         tools = _serialize_tools(api_client, input)
@@ -424,17 +425,3 @@ def get_path(obj: dict[str, Any], path: str, default: Any = None) -> Any | None:
         current = current[key]
 
     return current
-
-
-def _try_dict(obj: Any) -> dict[str, Any] | None:
-    try:
-        return obj.model_dump()
-    except AttributeError:
-        pass
-
-    try:
-        return obj.dump()
-    except AttributeError:
-        pass
-
-    return obj