deepeval 3.5.9__py3-none-any.whl → 3.6.1__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.5.9"
+__version__: str = "3.6.1"

deepeval/config/settings_manager.py

@@ -15,7 +15,7 @@ from enum import Enum
 from pydantic import SecretStr
 from deepeval.config.settings import get_settings, _SAVE_RE
 from deepeval.cli.dotenv_handler import DotenvHandler
-from deepeval.utils import bool_to_env_str
+from deepeval.config.utils import bool_to_env_str
 
 logger = logging.getLogger(__name__)
 StrOrEnum = Union[str, Enum]

deepeval/contextvars.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Optional
+
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden
+
+
+CURRENT_GOLDEN: ContextVar[Optional[Golden]] = ContextVar(
+    "CURRENT_GOLDEN", default=None
+)
+
+
+def set_current_golden(golden: Optional[Golden]):
+    return CURRENT_GOLDEN.set(golden)
+
+
+def get_current_golden() -> Optional[Golden]:
+    return CURRENT_GOLDEN.get()
+
+
+def reset_current_golden(token) -> None:
+    CURRENT_GOLDEN.reset(token)

deepeval/dataset/__init__.py

@@ -1,5 +1,11 @@
+from deepeval.contextvars import get_current_golden
 from .dataset import EvaluationDataset
 from .golden import Golden, ConversationalGolden
-from .test_run_tracer import init_global_test_run_tracer
 
-__all__ = ["EvaluationDataset", "Golden", "ConversationalGolden"]
+
+__all__ = [
+    "EvaluationDataset",
+    "Golden",
+    "ConversationalGolden",
+    "get_current_golden",
+]

deepeval/evaluate/execute.py

@@ -42,6 +42,7 @@ from deepeval.tracing.api import (
     BaseApiSpan,
 )
 from deepeval.dataset import Golden
+from deepeval.contextvars import set_current_golden, reset_current_golden
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.utils import copy_metrics
 from deepeval.utils import (
@@ -1480,6 +1481,7 @@ def execute_agentic_test_cases_from_loop(
         )
 
         for golden in goldens:
+            token = set_current_golden(golden)
             with capture_evaluation_run("golden"):
                 # yield golden
                 count += 1
@@ -1492,8 +1494,14 @@ def execute_agentic_test_cases_from_loop(
                     _progress=progress,
                     _pbar_callback_id=pbar_tags_id,
                 ):
-                    yield golden
-                    current_trace: Trace = current_trace_context.get()
+                    try:
+                        # yield golden to user code
+                        yield golden
+                        # control has returned from user code without error, capture trace now
+                        current_trace: Trace = current_trace_context.get()
+                    finally:
+                        # after user code returns control, always reset the context
+                        reset_current_golden(token)
 
                 update_pbar(progress, pbar_tags_id)
                 update_pbar(progress, pbar_id)
@@ -1849,6 +1857,7 @@ def a_execute_agentic_test_cases_from_loop(
 
         try:
             for index, golden in enumerate(goldens):
+                token = set_current_golden(golden)
                 current_golden_ctx.update(
                     {
                         "index": index,
@@ -1857,7 +1866,10 @@ def a_execute_agentic_test_cases_from_loop(
                     }
                 )
                 prev_task_length = len(created_tasks)
-                yield golden
+                try:
+                    yield golden
+                finally:
+                    reset_current_golden(token)
                 # if this golden created no tasks, bump bars now
                 if len(created_tasks) == prev_task_length:
                     update_pbar(progress, pbar_callback_id)

deepeval/openai_agents/__init__.py

@@ -1,6 +1,7 @@
 from deepeval.openai_agents.callback_handler import DeepEvalTracingProcessor
-from deepeval.openai_agents.runner import Runner
-from deepeval.openai_agents.patch import function_tool
 from deepeval.openai_agents.agent import DeepEvalAgent as Agent
+from deepeval.openai_agents.patch import function_tool
+
+# from deepeval.openai_agents.runner import Runner
 
-__all__ = ["DeepEvalTracingProcessor", "Runner", "function_tool", "Agent"]
+__all__ = ["DeepEvalTracingProcessor", "Agent", "function_tool"]

deepeval/openai_agents/agent.py

@@ -1,20 +1,17 @@
 from __future__ import annotations
 
-from dataclasses import dataclass, field, replace
-from typing import Any, Optional, Awaitable, Callable, Generic, TypeVar, List
+from dataclasses import dataclass
+from typing import Generic, TypeVar, List
 
-from deepeval.tracing import observe
 from deepeval.prompt import Prompt
-from deepeval.tracing.tracing import Observer
 from deepeval.metrics import BaseMetric
-from deepeval.tracing.utils import make_json_serializable
 from deepeval.tracing.types import LlmSpan
-from deepeval.tracing.context import current_span_context
 
 try:
     from agents.agent import Agent as BaseAgent
-    from agents.models.interface import Model, ModelProvider
-    from openai.types.responses import ResponseCompletedEvent
+    from deepeval.openai_agents.patch import (
+        patch_default_agent_runner_get_model,
+    )
 except Exception as e:
     raise RuntimeError(
         "openai-agents is required for this integration. Please install it."
@@ -23,163 +20,6 @@ except Exception as e:
 TContext = TypeVar("TContext")
 
 
-class _ObservedModel(Model):
-    def __init__(
-        self,
-        inner: Model,
-        llm_metric_collection: str = None,
-        llm_metrics: List[BaseMetric] = None,
-        confident_prompt: Prompt = None,
-    ) -> None:
-        self._inner = inner
-        self._llm_metric_collection = llm_metric_collection
-        self._llm_metrics = llm_metrics
-        self._confident_prompt = confident_prompt
-
-    def __getattr__(self, name: str) -> Any:
-        return getattr(self._inner, name)
-
-    def _get_model_name(self) -> str:
-        try:
-            for attr in ("model", "model_name", "name"):
-                if hasattr(self._inner, attr):
-                    val = getattr(self._inner, attr)
-                    if val is not None:
-                        return str(val)
-        except Exception:
-            pass
-        return "unknown"
-
-    async def get_response(
-        self,
-        system_instructions,
-        input,
-        model_settings,
-        tools,
-        output_schema,
-        handoffs,
-        tracing,
-        *,
-        previous_response_id,
-        conversation_id,
-        prompt,
-        **kwargs,
-    ):
-        model_name = self._get_model_name()
-        with Observer(
-            span_type="llm",
-            func_name="LLM",
-            function_kwargs={
-                "system_instructions": system_instructions,
-                "input": input,
-                "model_settings": model_settings,
-                "tools": tools,
-                "output_schema": output_schema,
-                "handoffs": handoffs,
-                # "tracing": tracing, # not important for llm spans
-                # "previous_response_id": previous_response_id, # not important for llm spans
-                # "conversation_id": conversation_id, # not important for llm spans
-                "prompt": prompt,
-                **kwargs,
-            },
-            observe_kwargs={"model": model_name},
-            metrics=self._llm_metrics,
-            metric_collection=self._llm_metric_collection,
-        ) as observer:
-            result = await self._inner.get_response(
-                system_instructions,
-                input,
-                model_settings,
-                tools,
-                output_schema,
-                handoffs,
-                tracing,
-                previous_response_id=previous_response_id,
-                conversation_id=conversation_id,
-                prompt=prompt,
-                **kwargs,
-            )
-            llm_span: LlmSpan = current_span_context.get()
-            llm_span.prompt = self._confident_prompt
-
-            observer.result = make_json_serializable(result.output)
-
-        return result
-
-    def stream_response(
-        self,
-        system_instructions,
-        input,
-        model_settings,
-        tools,
-        output_schema,
-        handoffs,
-        tracing,
-        *,
-        previous_response_id,
-        conversation_id,
-        prompt,
-        **kwargs,
-    ):
-        model_name = self._get_model_name()
-
-        async def _gen():
-            observer = Observer(
-                span_type="llm",
-                func_name="LLM",
-                function_kwargs={
-                    "system_instructions": system_instructions,
-                    "input": input,
-                    "model_settings": model_settings,
-                    "tools": tools,
-                    "output_schema": output_schema,
-                    "handoffs": handoffs,
-                    # "tracing": tracing,
-                    # "previous_response_id": previous_response_id,
-                    # "conversation_id": conversation_id,
-                    "prompt": prompt,
-                    **kwargs,
-                },
-                observe_kwargs={"model": model_name},
-                metrics=self._llm_metrics,
-                metric_collection=self._llm_metric_collection,
-            )
-            observer.__enter__()
-
-            llm_span: LlmSpan = current_span_context.get()
-            llm_span.prompt = self._confident_prompt
-
-            try:
-                async for event in self._inner.stream_response(
-                    system_instructions,
-                    input,
-                    model_settings,
-                    tools,
-                    output_schema,
-                    handoffs,
-                    tracing,
-                    previous_response_id=previous_response_id,
-                    conversation_id=conversation_id,
-                    prompt=prompt,
-                ):
-
-                    if isinstance(event, ResponseCompletedEvent):
-                        observer.result = make_json_serializable(
-                            event.response.output
-                        )
-
-                    yield event
-
-            except Exception as e:
-                observer.__exit__(type(e), e, e.__traceback__)
-                raise
-            finally:
-
-                observer.__exit__(None, None, None)
-
-        return _gen()
-
-
 @dataclass
 class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
     """
@@ -189,6 +29,8 @@ class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
     llm_metric_collection: str = None
     llm_metrics: List[BaseMetric] = None
     confident_prompt: Prompt = None
+    agent_metrics: List[BaseMetric] = None
+    agent_metric_collection: str = None
 
     def __post_init__(self):
-        super().__post_init__()
+        patch_default_agent_runner_get_model()

deepeval/openai_agents/callback_handler.py

@@ -21,6 +21,10 @@ try:
         ResponseSpanData,
         SpanData,
     )
+    from deepeval.openai_agents.patch import (
+        patch_default_agent_run_single_turn,
+        patch_default_agent_run_single_turn_streamed,
+    )
 
     openai_agents_available = True
 except ImportError:
@@ -37,6 +41,8 @@ def _check_openai_agents_available():
 class DeepEvalTracingProcessor(TracingProcessor):
     def __init__(self) -> None:
         _check_openai_agents_available()
+        patch_default_agent_run_single_turn()
+        patch_default_agent_run_single_turn_streamed()
         self.span_observers: dict[str, Observer] = {}
 
     def on_trace_start(self, trace: "Trace") -> None:
@@ -46,66 +52,69 @@ class DeepEvalTracingProcessor(TracingProcessor):
         _trace_name = trace_dict.get("workflow_name")
         _trace_metadata = trace_dict.get("metadata")
 
-        if _thread_id or _trace_metadata:
-            _trace = trace_manager.start_new_trace(trace_uuid=str(_trace_uuid))
-            _trace.thread_id = str(_thread_id)
-            _trace.name = str(_trace_name)
-            _trace.metadata = make_json_serializable(_trace_metadata)
-            current_trace_context.set(_trace)
-
-            trace_manager.add_span(  # adds a dummy root span
-                BaseSpan(
-                    uuid=_trace_uuid,
-                    trace_uuid=_trace_uuid,
-                    parent_uuid=None,
-                    start_time=perf_counter(),
-                    name=_trace_name,
-                    status=TraceSpanStatus.IN_PROGRESS,
-                    children=[],
-                )
+        _trace = trace_manager.start_new_trace(trace_uuid=str(_trace_uuid))
+        _trace.thread_id = str(_thread_id)
+        _trace.name = str(_trace_name)
+        _trace.metadata = make_json_serializable(_trace_metadata)
+        current_trace_context.set(_trace)
+
+        trace_manager.add_span(  # adds a dummy root span
+            BaseSpan(
+                uuid=_trace_uuid,
+                trace_uuid=_trace_uuid,
+                parent_uuid=None,
+                start_time=perf_counter(),
+                name=_trace_name,
+                status=TraceSpanStatus.IN_PROGRESS,
+                children=[],
             )
-        else:
-            current_trace = current_trace_context.get()
-            if current_trace:
-                current_trace.name = str(_trace_name)
+        )
 
     def on_trace_end(self, trace: "Trace") -> None:
         trace_dict = trace.export()
         _trace_uuid = trace_dict.get("id")
-        _thread_id = trace_dict.get("group_id")
         _trace_name = trace_dict.get("workflow_name")
-        _trace_metadata = trace_dict.get("metadata")
 
-        if _thread_id or _trace_metadata:
-            trace_manager.remove_span(
-                _trace_uuid
-            )  # removing the dummy root span
-            trace_manager.end_trace(_trace_uuid)
-            current_trace_context.set(None)
+        trace_manager.remove_span(_trace_uuid)  # removing the dummy root span
+        trace_manager.end_trace(_trace_uuid)
+        current_trace_context.set(None)
 
     def on_span_start(self, span: "Span") -> None:
         if not span.started_at:
             return
+        current_span = current_span_context.get()
+        if current_span and isinstance(
+            current_span, LlmSpan
+        ):  # llm span started by
+            return
+
         span_type = self.get_span_kind(span.span_data)
-        if span_type and span_type == "agent":
-            observer = Observer(span_type=span_type, func_name="NA")
-            observer.update_span_properties = (
-                lambda base_span: update_span_properties(
-                    base_span, span.span_data
-                )
-            )
-            self.span_observers[span.span_id] = observer
-            observer.__enter__()
+        observer = Observer(span_type=span_type, func_name="NA")
+        if span_type == "llm":
+            observer.observe_kwargs["model"] = "temporary model"
+        observer.update_span_properties = (
+            lambda span_type: update_span_properties(span_type, span.span_data)
+        )
+        self.span_observers[span.span_id] = observer
+        observer.__enter__()
 
     def on_span_end(self, span: "Span") -> None:
+        update_trace_properties_from_span_data(
+            current_trace_context.get(), span.span_data
+        )
+
         span_type = self.get_span_kind(span.span_data)
-        if span_type and span_type == "agent":
-            current_span = current_span_context.get()
-            if current_span:
-                update_span_properties(current_span, span.span_data)
-            observer = self.span_observers.pop(span.span_id, None)
-            if observer:
-                observer.__exit__(None, None, None)
+        current_span = current_span_context.get()
+        if (
+            current_span
+            and isinstance(current_span, LlmSpan)
+            and span_type == "llm"
+        ):  # addtional check if the span kind data is llm too
+            update_span_properties(current_span, span.span_data)
+
+        observer = self.span_observers.pop(span.span_id, None)
+        if observer:
+            observer.__exit__(None, None, None)
 
     def force_flush(self) -> None:
         pass
@@ -116,19 +125,18 @@ class DeepEvalTracingProcessor(TracingProcessor):
     def get_span_kind(self, span_data: "SpanData") -> str:
         if isinstance(span_data, AgentSpanData):
             return "agent"
-        # if isinstance(span_data, FunctionSpanData):
-        #     return "tool"
-        # if isinstance(span_data, MCPListToolsSpanData):
-        #     return "tool"
-        # if isinstance(span_data, GenerationSpanData):
-        #     return "llm"
-        # if isinstance(span_data, ResponseSpanData):
-        #     return "llm"
-        # if isinstance(span_data, HandoffSpanData):
-        #     return "custom"
-        # if isinstance(span_data, CustomSpanData):
-        #     return "base"
-        # if isinstance(span_data, GuardrailSpanData):
-        #     return "base"
-        # return "base"
-        return None
+        if isinstance(span_data, FunctionSpanData):
+            return "tool"
+        if isinstance(span_data, MCPListToolsSpanData):
+            return "tool"
+        if isinstance(span_data, GenerationSpanData):
+            return "llm"
+        if isinstance(span_data, ResponseSpanData):
+            return "llm"
+        if isinstance(span_data, HandoffSpanData):
+            return "custom"
+        if isinstance(span_data, CustomSpanData):
+            return "base"
+        if isinstance(span_data, GuardrailSpanData):
+            return "base"
+        return "base"

deepeval/openai_agents/extractors.py

@@ -1,9 +1,10 @@
+from deepeval.tracing.types import Trace
 from openai.types.responses.response_input_item_param import (
     FunctionCallOutput,
     Message,
 )
 from openai.types.responses.response_output_message_param import Content
-from typing import Union, List
+from typing import Union, List, Optional
 from openai.types.responses import (
     ResponseFunctionToolCallParam,
     ResponseOutputMessageParam,
@@ -25,6 +26,8 @@ from deepeval.tracing.types import (
 )
 import json
 
+from deepeval.tracing.utils import make_json_serializable
+
 try:
     from agents import MCPListToolsSpanData
     from agents.tracing.span_data import (
@@ -89,13 +92,17 @@ def update_span_properties_from_response_span_data(
         return
     # Extract usage tokens
     usage = response.usage
+    cached_input_tokens = None
+    ouptut_reasoning_tokens = None
     if usage:
         output_tokens = usage.output_tokens
         input_tokens = usage.input_tokens
         cached_input_tokens = usage.input_tokens_details.cached_tokens
         ouptut_reasoning_tokens = usage.output_tokens_details.reasoning_tokens
     # Get input and output
-    input = parse_response_input(span_data.input)
+    input = parse_response_input(
+        span_data.input, span_data.response.instructions
+    )
     raw_output = parse_response_output(response.output)
     output = (
         raw_output if isinstance(raw_output, str) else json.dumps(raw_output)
@@ -112,6 +119,23 @@ def update_span_properties_from_response_span_data(
     span.input = input
     span.output = output
     span.name = "LLM Generation"
+    response_dict = response.model_dump(exclude_none=True, mode="json")
+    span.metadata["invocation_params"] = {
+        k: v
+        for k, v in response_dict.items()
+        if k
+        in (
+            "max_output_tokens",
+            "parallel_tool_calls",
+            "reasoning",
+            "temperature",
+            "text",
+            "tool_choice",
+            "tools",
+            "top_p",
+            "truncation",
+        )
+    }
 
 
 def update_span_properties_from_generation_span_data(
@@ -136,6 +160,11 @@ def update_span_properties_from_generation_span_data(
     span.input = input
     span.output = output
     span.name = "LLM Generation"
+    span.metadata["invocation_params"] = {
+        "model_config": make_json_serializable(
+            generation_span_data.model_config
+        ),
+    }
 
 
 ########################################################
@@ -191,8 +220,6 @@ def update_span_properties_from_agent_span_data(
     if agent_span_data.output_type:
         metadata["output_type"] = agent_span_data.output_type
     span.metadata = metadata
-    span.input = None
-    span.output = None
 
 
 ########################################################
@@ -238,10 +265,30 @@ def update_span_properties_from_guardrail_span_data(
 ########################################################
 
 
-def parse_response_input(input: Union[str, List[ResponseInputItemParam]]):
-    if isinstance(input, str):
-        return input
+def parse_response_input(
+    input: Union[str, List[ResponseInputItemParam]],
+    instructions: Optional[Union[str, List[ResponseInputItemParam]]] = None,
+):
+
     processed_input = []
+
+    if isinstance(input, str) and isinstance(instructions, str):
+        return [
+            {"type": "message", "role": "system", "content": instructions},
+            {"type": "message", "role": "user", "content": input},
+        ]
+    elif isinstance(input, list) and isinstance(instructions, list):
+        input = instructions + input
+    elif isinstance(input, list) and isinstance(instructions, str):
+        processed_input += [
+            {"type": "message", "role": "system", "content": instructions}
+        ]
+    elif isinstance(input, str) and isinstance(instructions, list):
+        processed_input += [
+            {"type": "message", "role": "user", "content": input}
+        ]
+        input = instructions
+
     for item in input:
         if "type" not in item:
             if "role" in item and "content" in item:
@@ -365,3 +412,32 @@ def parse_function_call(
         "name": function_call.name,
         "arguments": function_call.arguments,
     }
+
+
+def update_trace_properties_from_span_data(
+    trace: Trace,
+    span_data: Union["ResponseSpanData", "GenerationSpanData"],
+):
+    if isinstance(span_data, ResponseSpanData):
+        if not trace.input:
+            trace.input = parse_response_input(
+                span_data.input, span_data.response.instructions
+            )
+        raw_output = parse_response_output(span_data.response.output)
+        output = (
+            raw_output
+            if isinstance(raw_output, str)
+            else json.dumps(raw_output)
+        )
+        trace.output = output
+
+    elif isinstance(span_data, GenerationSpanData):
+        if not trace.input:
+            trace.input = span_data.input
+        raw_output = span_data.output
+        output = (
+            raw_output
+            if isinstance(raw_output, str)
+            else json.dumps(raw_output)
+        )
+        trace.output = output