deepeval 3.5.3__py3-none-any.whl → 3.5.5__py3-none-any.whl

deepeval/integrations/pydantic_ai/utils.py

@@ -1,8 +1,29 @@
-from typing import List
-from pydantic_ai.messages import ModelResponsePart
+from time import perf_counter
+from contextlib import asynccontextmanager
+import inspect
+import functools
+from typing import Any, Callable, List, Optional
+
+from pydantic_ai.models import Model
 from pydantic_ai.agent import AgentRunResult
 from pydantic_ai._run_context import RunContext
+from pydantic_ai.messages import (
+    ModelRequest,
+    ModelResponse,
+    ModelResponsePart,
+    SystemPromptPart,
+    TextPart,
+    ToolCallPart,
+    ToolReturnPart,
+    UserPromptPart,
+)
+
+from deepeval.prompt import Prompt
+from deepeval.tracing.tracing import Observer
+from deepeval.metrics.base_metric import BaseMetric
 from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import current_trace_context, current_span_context
+from deepeval.tracing.types import AgentSpan, LlmOutput, LlmSpan, LlmToolCall
 
 
 # llm tools called
@@ -84,3 +105,219 @@ def sanitize_run_context(value):
         return {sanitize_run_context(v) for v in value}
 
     return value
+
+
+def patch_llm_model(
+    model: Model,
+    llm_metric_collection: Optional[str] = None,
+    llm_metrics: Optional[List[BaseMetric]] = None,
+    llm_prompt: Optional[Prompt] = None,
+):
+    original_func = model.request
+    sig = inspect.signature(original_func)
+
+    try:
+        model_name = model.model_name
+    except Exception:
+        model_name = "unknown"
+
+    @functools.wraps(original_func)
+    async def wrapper(*args, **kwargs):
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        request = bound.arguments.get("messages", [])
+
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            observe_kwargs={"model": model_name},
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+        ) as observer:
+            result = await original_func(*args, **kwargs)
+            observer.update_span_properties = (
+                lambda llm_span: set_llm_span_attributes(
+                    llm_span, request, result, llm_prompt
+                )
+            )
+            observer.result = result
+            return result
+
+    model.request = wrapper
+
+    stream_original_func = model.request_stream
+    stream_sig = inspect.signature(stream_original_func)
+
+    @asynccontextmanager
+    async def stream_wrapper(*args, **kwargs):
+        bound = stream_sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        request = bound.arguments.get("messages", [])
+
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            observe_kwargs={"model": model_name},
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+        ) as observer:
+            llm_span: LlmSpan = current_span_context.get()
+            async with stream_original_func(
+                *args, **kwargs
+            ) as streamed_response:
+                try:
+                    yield streamed_response
+                    if not llm_span.token_intervals:
+                        llm_span.token_intervals = {perf_counter(): "NA"}
+                    else:
+                        llm_span.token_intervals[perf_counter()] = "NA"
+                finally:
+                    try:
+                        result = streamed_response.get()
+                        observer.update_span_properties = (
+                            lambda llm_span: set_llm_span_attributes(
+                                llm_span, request, result, llm_prompt
+                            )
+                        )
+                        observer.result = result
+                    except Exception:
+                        pass
+
+    model.request_stream = stream_wrapper
+
+
+def create_patched_tool(
+    func: Callable,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
+    import asyncio
+
+    original_func = func
+
+    is_async = asyncio.iscoroutinefunction(original_func)
+
+    if is_async:
+
+        @functools.wraps(original_func)
+        async def async_wrapper(*args, **kwargs):
+            sanitized_args = sanitize_run_context(args)
+            sanitized_kwargs = sanitize_run_context(kwargs)
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": sanitized_args, **sanitized_kwargs},
+            ) as observer:
+                result = await original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return async_wrapper
+    else:
+
+        @functools.wraps(original_func)
+        def sync_wrapper(*args, **kwargs):
+            sanitized_args = sanitize_run_context(args)
+            sanitized_kwargs = sanitize_run_context(kwargs)
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": sanitized_args, **sanitized_kwargs},
+            ) as observer:
+                result = original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return sync_wrapper
+
+
+def update_trace_context(
+    trace_name: Optional[str] = None,
+    trace_tags: Optional[List[str]] = None,
+    trace_metadata: Optional[dict] = None,
+    trace_thread_id: Optional[str] = None,
+    trace_user_id: Optional[str] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+    trace_input: Optional[Any] = None,
+    trace_output: Optional[Any] = None,
+):
+
+    current_trace = current_trace_context.get()
+
+    if trace_name:
+        current_trace.name = trace_name
+    if trace_tags:
+        current_trace.tags = trace_tags
+    if trace_metadata:
+        current_trace.metadata = trace_metadata
+    if trace_thread_id:
+        current_trace.thread_id = trace_thread_id
+    if trace_user_id:
+        current_trace.user_id = trace_user_id
+    if trace_metric_collection:
+        current_trace.metric_collection = trace_metric_collection
+    if trace_metrics:
+        current_trace.metrics = trace_metrics
+    if trace_input:
+        current_trace.input = trace_input
+    if trace_output:
+        current_trace.output = trace_output
+
+
+def set_llm_span_attributes(
+    llm_span: LlmSpan,
+    requests: List[ModelRequest],
+    result: ModelResponse,
+    llm_prompt: Optional[Prompt] = None,
+):
+    llm_span.prompt = llm_prompt
+
+    input = []
+    for request in requests:
+        for part in request.parts:
+            if isinstance(part, SystemPromptPart):
+                input.append({"role": "System", "content": part.content})
+            elif isinstance(part, UserPromptPart):
+                input.append({"role": "User", "content": part.content})
+            elif isinstance(part, ToolCallPart):
+                input.append(
+                    {
+                        "role": "Tool Call",
+                        "name": part.tool_name,
+                        "content": part.args_as_json_str(),
+                    }
+                )
+            elif isinstance(part, ToolReturnPart):
+                input.append(
+                    {
+                        "role": "Tool Return",
+                        "name": part.tool_name,
+                        "content": part.model_response_str(),
+                    }
+                )
+    llm_span.input = input
+
+    content = ""
+    tool_calls = []
+    for part in result.parts:
+        if isinstance(part, TextPart):
+            content += part.content + "\n"
+        elif isinstance(part, ToolCallPart):
+            tool_calls.append(
+                LlmToolCall(name=part.tool_name, args=part.args_as_dict())
+            )
+    llm_span.output = LlmOutput(
+        role="Assistant", content=content, tool_calls=tool_calls
+    )
+    llm_span.tools_called = extract_tools_called_from_llm_response(result.parts)
+
+
+def set_agent_span_attributes(agent_span: AgentSpan, result: AgentRunResult):
+    agent_span.tools_called = extract_tools_called(result)

deepeval/metrics/mcp_use_metric/mcp_use_metric.py

@@ -283,8 +283,9 @@ class MCPUseMetric(BaseMetric):
         mcp_resources_called: List[MCPResourceCall],
         mcp_prompts_called: List[MCPPromptCall],
     ) -> tuple[str, str]:
+        available_primitives = "MCP Primitives Available: \n"
         for mcp_server in mcp_servers:
-            available_primitives = f"MCP Server {mcp_server.server_name}\n"
+            available_primitives += f"MCP Server {mcp_server.server_name}\n"
             available_primitives += (
                 (
                     "\nAvailable Tools:\n[\n"

deepeval/metrics/non_advice/non_advice.py

@@ -43,7 +43,7 @@ class NonAdviceMetric(BaseMetric):
                 "or ['financial', 'medical'] for multiple types."
             )
 
-        self.threshold = 0 if strict_mode else threshold
+        self.threshold = 1 if strict_mode else threshold
         self.advice_types = advice_types
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
@@ -293,7 +293,7 @@ class NonAdviceMetric(BaseMetric):
                 appropriate_advice_count += 1
 
         score = appropriate_advice_count / number_of_verdicts
-        return 1 if self.strict_mode and score < 1 else score
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -35,7 +35,7 @@ class PIILeakageMetric(BaseMetric):
         verbose_mode: bool = False,
         evaluation_template: Type[PIILeakageTemplate] = PIILeakageTemplate,
     ):
-        self.threshold = 0 if strict_mode else threshold
+        self.threshold = 1 if strict_mode else threshold
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.include_reason = include_reason

deepeval/openai_agents/agent.py

@@ -1,14 +1,20 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field, replace
-from typing import Any, Optional, Awaitable, Callable, Generic, TypeVar
+from typing import Any, Optional, Awaitable, Callable, Generic, TypeVar, List
 
 from deepeval.tracing import observe
 from deepeval.prompt import Prompt
+from deepeval.tracing.tracing import Observer
+from deepeval.metrics import BaseMetric
+from deepeval.tracing.utils import make_json_serializable
+from deepeval.tracing.types import LlmSpan
+from deepeval.tracing.context import current_span_context
 
 try:
     from agents.agent import Agent as BaseAgent
     from agents.models.interface import Model, ModelProvider
+    from openai.types.responses import ResponseCompletedEvent
 except Exception as e:
     raise RuntimeError(
         "openai-agents is required for this integration. Please install it."
@@ -21,17 +27,15 @@ class _ObservedModel(Model):
     def __init__(
         self,
         inner: Model,
-        *,
-        metrics: Optional[list[Any]] = None,
-        metric_collection: Optional[str] = None,
-        deepeval_prompt: Optional[Any] = None,
+        llm_metric_collection: str = None,
+        llm_metrics: List[BaseMetric] = None,
+        confident_prompt: Prompt = None,
     ) -> None:
         self._inner = inner
-        self._metrics = metrics
-        self._metric_collection = metric_collection
-        self._deepeval_prompt = deepeval_prompt
+        self._llm_metric_collection = llm_metric_collection
+        self._llm_metrics = llm_metrics
+        self._confident_prompt = confident_prompt
 
-    # Delegate attributes not overridden
     def __getattr__(self, name: str) -> Any:
         return getattr(self._inner, name)
 
@@ -59,29 +63,48 @@ class _ObservedModel(Model):
         previous_response_id,
         conversation_id,
         prompt,
+        **kwargs,
     ):
         model_name = self._get_model_name()
-
-        wrapped = observe(
-            metrics=self._metrics,
-            metric_collection=self._metric_collection,
-            type="llm",
-            model=model_name,
-            prompt=self._deepeval_prompt,
-        )(self._inner.get_response)
-
-        return await wrapped(
-            system_instructions,
-            input,
-            model_settings,
-            tools,
-            output_schema,
-            handoffs,
-            tracing,
-            previous_response_id=previous_response_id,
-            conversation_id=conversation_id,
-            prompt=prompt,
-        )
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            function_kwargs={
+                "system_instructions": system_instructions,
+                "input": input,
+                "model_settings": model_settings,
+                "tools": tools,
+                "output_schema": output_schema,
+                "handoffs": handoffs,
+                # "tracing": tracing, # not important for llm spans
+                # "previous_response_id": previous_response_id, # not important for llm spans
+                # "conversation_id": conversation_id, # not important for llm spans
+                "prompt": prompt,
+                **kwargs,
+            },
+            observe_kwargs={"model": model_name},
+            metrics=self._llm_metrics,
+            metric_collection=self._llm_metric_collection,
+        ) as observer:
+            result = await self._inner.get_response(
+                system_instructions,
+                input,
+                model_settings,
+                tools,
+                output_schema,
+                handoffs,
+                tracing,
+                previous_response_id=previous_response_id,
+                conversation_id=conversation_id,
+                prompt=prompt,
+                **kwargs,
+            )
+            llm_span: LlmSpan = current_span_context.get()
+            llm_span.prompt = self._confident_prompt
+
+            observer.result = make_json_serializable(result.output)
+
+        return result
 
     def stream_response(
         self,
@@ -96,91 +119,77 @@ class _ObservedModel(Model):
         previous_response_id,
         conversation_id,
         prompt,
+        **kwargs,
     ):
-        # Optional: if you also want to observe streaming, uncomment and wrap similarly.
-        # wrapped = observe(
-        #     metrics=self._metrics,
-        #     metric_collection=self._metric_collection,
-        #     type="llm",
-        #     model=model_name,
-        # )(self._inner.stream_response)
-        # return wrapped(
-        #     system_instructions,
-        #     input,
-        #     model_settings,
-        #     tools,
-        #     output_schema,
-        #     handoffs,
-        #     tracing,
-        #     previous_response_id=previous_response_id,
-        #     conversation_id=conversation_id,
-        #     prompt=prompt,
-        # )
-        return self._inner.stream_response(
-            system_instructions,
-            input,
-            model_settings,
-            tools,
-            output_schema,
-            handoffs,
-            tracing,
-            previous_response_id=previous_response_id,
-            conversation_id=conversation_id,
-            prompt=prompt,
-        )
-
-
-class _ObservedProvider(ModelProvider):
-    def __init__(
-        self,
-        base: ModelProvider,
-        *,
-        metrics: Optional[list[Any]] = None,
-        metric_collection: Optional[str] = None,
-        deepeval_prompt: Optional[Any] = None,
-    ) -> None:
-        self._base = base
-        self._metrics = metrics
-        self._metric_collection = metric_collection
-        self._deepeval_prompt = deepeval_prompt
+        model_name = self._get_model_name()
 
-    def get_model(self, model_name: str | None) -> Model:
-        model = self._base.get_model(model_name)
-        return _ObservedModel(
-            model,
-            metrics=self._metrics,
-            metric_collection=self._metric_collection,
-            deepeval_prompt=self._deepeval_prompt,
-        )
+        async def _gen():
+            observer = Observer(
+                span_type="llm",
+                func_name="LLM",
+                function_kwargs={
+                    "system_instructions": system_instructions,
+                    "input": input,
+                    "model_settings": model_settings,
+                    "tools": tools,
+                    "output_schema": output_schema,
+                    "handoffs": handoffs,
+                    # "tracing": tracing,
+                    # "previous_response_id": previous_response_id,
+                    # "conversation_id": conversation_id,
+                    "prompt": prompt,
+                    **kwargs,
+                },
+                observe_kwargs={"model": model_name},
+                metrics=self._llm_metrics,
+                metric_collection=self._llm_metric_collection,
+            )
+            observer.__enter__()
+
+            llm_span: LlmSpan = current_span_context.get()
+            llm_span.prompt = self._confident_prompt
+
+            try:
+                async for event in self._inner.stream_response(
+                    system_instructions,
+                    input,
+                    model_settings,
+                    tools,
+                    output_schema,
+                    handoffs,
+                    tracing,
+                    previous_response_id=previous_response_id,
+                    conversation_id=conversation_id,
+                    prompt=prompt,
+                ):
+
+                    if isinstance(event, ResponseCompletedEvent):
+                        observer.result = (
+                            event.response.output_text
+                        )  # TODO: support other response types
+
+                    yield event
+
+                observer.__exit__(None, None, None)
+            except Exception as e:
+                observer.__exit__(type(e), e, e.__traceback__)
+                raise
+            finally:
+
+                observer.__exit__(None, None, None)
+
+        return _gen()
 
 
 @dataclass
 class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
     """
-    A subclass of agents.Agent that accepts `metrics` and `metric_collection`
-    and ensures the underlying model's `get_response` is wrapped with deepeval.observe.
+    A subclass of agents.Agent.
     """
 
-    metrics: list[Any] | None = field(default=None)
-    metric_collection: str | None = field(default=None)
-    deepeval_prompt: Prompt | None = field(default=None)
+    llm_metric_collection: str = None
+    llm_metrics: List[BaseMetric] = None
+    confident_prompt: Prompt = None
 
     def __post_init__(self):
         super().__post_init__()
-        # If a direct Model instance is set on the agent, wrap it here.
-        if self.model is not None and not isinstance(self.model, str):
-            try:
-                from agents.models.interface import (
-                    Model as _Model,
-                )  # local import for safety
-
-                if isinstance(self.model, _Model):
-                    self.model = _ObservedModel(
-                        self.model,
-                        metrics=self.metrics,
-                        metric_collection=self.metric_collection,
-                        deepeval_prompt=self.deepeval_prompt,
-                    )
-            except Exception:
-                # If we can't import or wrap, silently skip.
-                pass

deepeval/openai_agents/callback_handler.py

@@ -46,17 +46,7 @@ class DeepEvalTracingProcessor(TracingProcessor):
         if not span.started_at:
             return
         span_type = self.get_span_kind(span.span_data)
-        if span_type == "agent":
-            if isinstance(span.span_data, AgentSpanData):
-                current_trace = current_trace_context.get()
-                if current_trace:
-                    current_trace.name = span.span_data.name
-
-        if span_type == "tool":
-            return
-        elif span_type == "llm":
-            return
-        else:
+        if span_type and span_type == "agent":
             observer = Observer(span_type=span_type, func_name="NA")
             observer.update_span_properties = (
                 lambda base_span: update_span_properties(
@@ -68,13 +58,13 @@ class DeepEvalTracingProcessor(TracingProcessor):
 
     def on_span_end(self, span: "Span") -> None:
         span_type = self.get_span_kind(span.span_data)
-        if span_type == "llm":
+        if span_type and span_type == "agent":
             current_span = current_span_context.get()
             if current_span:
                 update_span_properties(current_span, span.span_data)
-        observer = self.span_observers.pop(span.span_id, None)
-        if observer:
-            observer.__exit__(None, None, None)
+            observer = self.span_observers.pop(span.span_id, None)
+            if observer:
+                observer.__exit__(None, None, None)
 
     def force_flush(self) -> None:
         pass
@@ -85,18 +75,19 @@ class DeepEvalTracingProcessor(TracingProcessor):
     def get_span_kind(self, span_data: "SpanData") -> str:
         if isinstance(span_data, AgentSpanData):
             return "agent"
-        if isinstance(span_data, FunctionSpanData):
-            return "tool"
-        if isinstance(span_data, MCPListToolsSpanData):
-            return "tool"
-        if isinstance(span_data, GenerationSpanData):
-            return "llm"
-        if isinstance(span_data, ResponseSpanData):
-            return "llm"
-        if isinstance(span_data, HandoffSpanData):
-            return "custom"
-        if isinstance(span_data, CustomSpanData):
-            return "base"
-        if isinstance(span_data, GuardrailSpanData):
-            return "base"
-        return "base"
+        # if isinstance(span_data, FunctionSpanData):
+        #     return "tool"
+        # if isinstance(span_data, MCPListToolsSpanData):
+        #     return "tool"
+        # if isinstance(span_data, GenerationSpanData):
+        #     return "llm"
+        # if isinstance(span_data, ResponseSpanData):
+        #     return "llm"
+        # if isinstance(span_data, HandoffSpanData):
+        #     return "custom"
+        # if isinstance(span_data, CustomSpanData):
+        #     return "base"
+        # if isinstance(span_data, GuardrailSpanData):
+        #     return "base"
+        # return "base"
+        return None