deepeval 3.5.1__py3-none-any.whl → 3.5.3__py3-none-any.whl

deepeval/integrations/pydantic_ai/patcher.py

@@ -12,6 +12,8 @@ from deepeval.confident.api import get_confident_api_key
 from deepeval.integrations.pydantic_ai.otel import instrument_pydantic_ai
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.prompt import Prompt
+import inspect
+from contextvars import ContextVar
 
 try:
     from pydantic_ai.agent import Agent
@@ -26,11 +28,68 @@ try:
         ToolReturnPart,
         UserPromptPart,
     )
+    from pydantic_ai._run_context import RunContext
+    from deepeval.integrations.pydantic_ai.utils import (
+        extract_tools_called_from_llm_response,
+        extract_tools_called,
+        sanitize_run_context,
+    )
 
     pydantic_ai_installed = True
 except:
     pydantic_ai_installed = True
 
+_IN_RUN_SYNC = ContextVar("deepeval_in_run_sync", default=False)
+_INSTRUMENTED = False
+
+
+def instrument(otel: Optional[bool] = False, api_key: Optional[str] = None):
+    global _INSTRUMENTED
+    if api_key:
+        deepeval.login(api_key)
+
+    api_key = get_confident_api_key()
+
+    if not api_key:
+        raise ValueError("No api key provided.")
+
+    if otel:
+        instrument_pydantic_ai(api_key)
+    else:
+        with capture_tracing_integration("pydantic_ai"):
+            if _INSTRUMENTED:
+                return
+            _patch_agent_init()
+            _patch_agent_tool_decorator()
+            _INSTRUMENTED = True
+
+
+################### Init Patches ###################
+
+
+def _patch_agent_init():
+    original_init = Agent.__init__
+
+    @functools.wraps(original_init)
+    def wrapper(
+        *args,
+        llm_metric_collection: Optional[str] = None,
+        llm_metrics: Optional[List[BaseMetric]] = None,
+        llm_prompt: Optional[Prompt] = None,
+        agent_metric_collection: Optional[str] = None,
+        agent_metrics: Optional[List[BaseMetric]] = None,
+        **kwargs
+    ):
+        result = original_init(*args, **kwargs)
+        _patch_llm_model(
+            args[0]._model, llm_metric_collection, llm_metrics, llm_prompt
+        )  # runtime patch of the model
+        _patch_agent_run(args[0], agent_metric_collection, agent_metrics)
+        _patch_agent_run_sync(args[0], agent_metric_collection, agent_metrics)
+        return result
+
+    Agent.__init__ = wrapper
+
 
 def _patch_agent_tool_decorator():
     original_tool = Agent.tool
@@ -64,101 +123,101 @@ def _patch_agent_tool_decorator():
     Agent.tool = wrapper
 
 
-def _create_patched_tool(
-    func: Callable,
-    metrics: Optional[List[BaseMetric]] = None,
-    metric_collection: Optional[str] = None,
-):
-    import asyncio
-
-    original_func = func
-
-    is_async = asyncio.iscoroutinefunction(original_func)
+################### Runtime Patches ###################
 
-    if is_async:
 
-        @functools.wraps(original_func)
-        async def async_wrapper(*args, **kwargs):
-            with Observer(
-                span_type="tool",
-                func_name=original_func.__name__,
-                metrics=metrics,
-                metric_collection=metric_collection,
-                function_kwargs={"args": args, **kwargs},
-            ) as observer:
-                result = await original_func(*args, **kwargs)
-                observer.result = result
-
-            return result
+def _patch_agent_run_sync(
+    agent: Agent,
+    agent_metric_collection: Optional[str] = None,
+    agent_metrics: Optional[List[BaseMetric]] = None,
+):
+    original_run_sync = agent.run_sync
 
-        return async_wrapper
-    else:
+    @functools.wraps(original_run_sync)
+    def wrapper(
+        *args,
+        metric_collection: Optional[str] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs
+    ):
 
-        @functools.wraps(original_func)
-        def sync_wrapper(*args, **kwargs):
-            with Observer(
-                span_type="tool",
-                func_name=original_func.__name__,
-                metrics=metrics,
-                metric_collection=metric_collection,
-                function_kwargs={"args": args, **kwargs},
-            ) as observer:
-                result = original_func(*args, **kwargs)
-                observer.result = result
+        sig = inspect.signature(original_run_sync)
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        input = bound.arguments.get("user_prompt", None)
 
-            return result
+        with Observer(
+            span_type="agent",
+            func_name="Agent",
+            function_kwargs={"input": input},
+            metrics=agent_metrics,
+            metric_collection=agent_metric_collection,
+        ) as observer:
 
-        return sync_wrapper
+            token = _IN_RUN_SYNC.set(True)
+            try:
+                result = original_run_sync(*args, **kwargs)
+            finally:
+                _IN_RUN_SYNC.reset(token)
 
+            observer.update_span_properties = (
+                lambda agent_span: set_agent_span_attributes(agent_span, result)
+            )
+            observer.result = result.output
 
-def _patch_agent_init():
-    original_init = Agent.__init__
+            _update_trace_context(
+                trace_name=name,
+                trace_tags=tags,
+                trace_metadata=metadata,
+                trace_thread_id=thread_id,
+                trace_user_id=user_id,
+                trace_metric_collection=metric_collection,
+                trace_metrics=metrics,
+                trace_input=input,
+                trace_output=result.output,
+            )
 
-    @functools.wraps(original_init)
-    def wrapper(
-        self,
-        *args,
-        llm_metric_collection: Optional[str] = None,
-        llm_metrics: Optional[List[BaseMetric]] = None,
-        llm_prompt: Optional[Prompt] = None,
-        agent_metric_collection: Optional[str] = None,
-        agent_metrics: Optional[List[BaseMetric]] = None,
-        **kwargs
-    ):
-        result = original_init(self, *args, **kwargs)
-        _patch_llm_model(
-            self._model, llm_metric_collection, llm_metrics, llm_prompt
-        )  # runtime patch of the model
-        _patch_agent_run(agent_metric_collection, agent_metrics)
         return result
 
-    Agent.__init__ = wrapper
+    agent.run_sync = wrapper
 
 
 def _patch_agent_run(
+    agent: Agent,
     agent_metric_collection: Optional[str] = None,
     agent_metrics: Optional[List[BaseMetric]] = None,
 ):
-    original_run = Agent.run
+    original_run = agent.run
 
     @functools.wraps(original_run)
     async def wrapper(
         *args,
-        trace_metric_collection: Optional[str] = None,
-        trace_metrics: Optional[List[BaseMetric]] = None,
-        trace_name: Optional[str] = None,
-        trace_tags: Optional[List[str]] = None,
-        trace_metadata: Optional[dict] = None,
-        trace_thread_id: Optional[str] = None,
-        trace_user_id: Optional[str] = None,
+        metric_collection: Optional[str] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
         **kwargs
     ):
+        sig = inspect.signature(original_run)
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        input = bound.arguments.get("user_prompt", None)
+
+        in_sync = _IN_RUN_SYNC.get()
         with Observer(
-            span_type="agent",
-            func_name="Agent",
-            function_kwargs={"input": args[1]},
-            metrics=agent_metrics,
-            metric_collection=agent_metric_collection,
+            span_type="agent" if not in_sync else "custom",
+            func_name="Agent" if not in_sync else "run",
+            function_kwargs={"input": input},
+            metrics=agent_metrics if not in_sync else None,
+            metric_collection=agent_metric_collection if not in_sync else None,
         ) as observer:
             result = await original_run(*args, **kwargs)
             observer.update_span_properties = (
@@ -167,44 +226,20 @@ def _patch_agent_run(
             observer.result = result.output
 
             _update_trace_context(
-                trace_name=trace_name,
-                trace_tags=trace_tags,
-                trace_metadata=trace_metadata,
-                trace_thread_id=trace_thread_id,
-                trace_user_id=trace_user_id,
-                trace_metric_collection=trace_metric_collection,
-                trace_metrics=trace_metrics,
-                trace_input=args[1],
+                trace_name=name,
+                trace_tags=tags,
+                trace_metadata=metadata,
+                trace_thread_id=thread_id,
+                trace_user_id=user_id,
+                trace_metric_collection=metric_collection,
+                trace_metrics=metrics,
+                trace_input=input,
                 trace_output=result.output,
             )
 
         return result
 
-    Agent.run = wrapper
-
-
-def _update_trace_context(
-    trace_name: Optional[str] = None,
-    trace_tags: Optional[List[str]] = None,
-    trace_metadata: Optional[dict] = None,
-    trace_thread_id: Optional[str] = None,
-    trace_user_id: Optional[str] = None,
-    trace_metric_collection: Optional[str] = None,
-    trace_metrics: Optional[List[BaseMetric]] = None,
-    trace_input: Optional[Any] = None,
-    trace_output: Optional[Any] = None,
-):
-
-    current_trace = current_trace_context.get()
-    current_trace.name = trace_name
-    current_trace.tags = trace_tags
-    current_trace.metadata = trace_metadata
-    current_trace.thread_id = trace_thread_id
-    current_trace.user_id = trace_user_id
-    current_trace.metric_collection = trace_metric_collection
-    current_trace.metrics = trace_metrics
-    current_trace.input = trace_input
-    current_trace.output = trace_output
+    agent.run = wrapper
 
 
 def _patch_llm_model(
@@ -214,6 +249,8 @@ def _patch_llm_model(
     llm_prompt: Optional[Prompt] = None,
 ):
     original_func = model.request
+    sig = inspect.signature(original_func)
+
     try:
         model_name = model.model_name
     except Exception:
@@ -221,6 +258,10 @@ def _patch_llm_model(
 
     @functools.wraps(original_func)
     async def wrapper(*args, **kwargs):
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        request = bound.arguments.get("messages", [])
+
         with Observer(
             span_type="llm",
             func_name="LLM",
@@ -229,36 +270,93 @@ def _patch_llm_model(
             metric_collection=llm_metric_collection,
         ) as observer:
             result = await original_func(*args, **kwargs)
-            request = kwargs.get("messages", [])
-            if not request:
-                request = args[0]
             observer.update_span_properties = (
                 lambda llm_span: set_llm_span_attributes(
-                    llm_span, args[0], result, llm_prompt
+                    llm_span, request, result, llm_prompt
                 )
             )
             observer.result = result
-        return result
+            return result
 
     model.request = wrapper
 
 
-def instrument(otel: Optional[bool] = False, api_key: Optional[str] = None):
+################### Helper Functions ###################
 
-    if api_key:
-        deepeval.login(api_key)
 
-    api_key = get_confident_api_key()
+def _create_patched_tool(
+    func: Callable,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
+    import asyncio
 
-    if not api_key:
-        raise ValueError("No api key provided.")
+    original_func = func
 
-    if otel:
-        instrument_pydantic_ai(api_key)
+    is_async = asyncio.iscoroutinefunction(original_func)
+
+    if is_async:
+
+        @functools.wraps(original_func)
+        async def async_wrapper(*args, **kwargs):
+            sanitized_args = sanitize_run_context(args)
+            sanitized_kwargs = sanitize_run_context(kwargs)
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": sanitized_args, **sanitized_kwargs},
+            ) as observer:
+                result = await original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return async_wrapper
     else:
-        with capture_tracing_integration("pydantic_ai"):
-            _patch_agent_init()
-            _patch_agent_tool_decorator()
+
+        @functools.wraps(original_func)
+        def sync_wrapper(*args, **kwargs):
+            sanitized_args = sanitize_run_context(args)
+            sanitized_kwargs = sanitize_run_context(kwargs)
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": sanitized_args, **sanitized_kwargs},
+            ) as observer:
+                result = original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return sync_wrapper
+
+
+def _update_trace_context(
+    trace_name: Optional[str] = None,
+    trace_tags: Optional[List[str]] = None,
+    trace_metadata: Optional[dict] = None,
+    trace_thread_id: Optional[str] = None,
+    trace_user_id: Optional[str] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+    trace_input: Optional[Any] = None,
+    trace_output: Optional[Any] = None,
+):
+
+    current_trace = current_trace_context.get()
+    current_trace.name = trace_name
+    current_trace.tags = trace_tags
+    current_trace.metadata = trace_metadata
+    current_trace.thread_id = trace_thread_id
+    current_trace.user_id = trace_user_id
+    current_trace.metric_collection = trace_metric_collection
+    current_trace.metrics = trace_metrics
+    current_trace.input = trace_input
+    current_trace.output = trace_output
 
 
 def set_llm_span_attributes(
@@ -306,71 +404,8 @@ def set_llm_span_attributes(
     llm_span.output = LlmOutput(
         role="Assistant", content=content, tool_calls=tool_calls
     )
-    llm_span.tools_called = _extract_tools_called_from_llm_response(
-        result.parts
-    )
+    llm_span.tools_called = extract_tools_called_from_llm_response(result.parts)
 
 
 def set_agent_span_attributes(agent_span: AgentSpan, result: AgentRunResult):
-    agent_span.tools_called = _extract_tools_called(result)
-
-
-# llm tools called
-def _extract_tools_called_from_llm_response(
-    result: List[ModelResponsePart],
-) -> List[ToolCall]:
-    tool_calls = []
-
-    # Loop through each ModelResponsePart
-    for part in result:
-        # Look for parts with part_kind="tool-call"
-        if hasattr(part, "part_kind") and part.part_kind == "tool-call":
-            # Extract tool name and args from the ToolCallPart
-            tool_name = part.tool_name
-            input_parameters = (
-                part.args_as_dict() if hasattr(part, "args_as_dict") else None
-            )
-
-            # Create and append ToolCall object
-            tool_call = ToolCall(
-                name=tool_name, input_parameters=input_parameters
-            )
-            tool_calls.append(tool_call)
-
-    return tool_calls
-
-
-# TODO: llm tools called (reposne is present next message)
-def _extract_tools_called(result: AgentRunResult) -> List[ToolCall]:
-    tool_calls = []
-
-    # Access the message history from the _state
-    message_history = result._state.message_history
-
-    # Scan through all messages in the history
-    for message in message_history:
-        # Check if this is a ModelResponse (kind="response")
-        if hasattr(message, "kind") and message.kind == "response":
-            # For ModelResponse messages, check each part
-            if hasattr(message, "parts"):
-                for part in message.parts:
-                    # Look for parts with part_kind="tool-call"
-                    if (
-                        hasattr(part, "part_kind")
-                        and part.part_kind == "tool-call"
-                    ):
-                        # Extract tool name and args from the ToolCallPart
-                        tool_name = part.tool_name
-                        input_parameters = (
-                            part.args_as_dict()
-                            if hasattr(part, "args_as_dict")
-                            else None
-                        )
-
-                        # Create and append ToolCall object
-                        tool_call = ToolCall(
-                            name=tool_name, input_parameters=input_parameters
-                        )
-                        tool_calls.append(tool_call)
-
-    return tool_calls
+    agent_span.tools_called = extract_tools_called(result)

deepeval/integrations/pydantic_ai/utils.py

@@ -0,0 +1,86 @@
+from typing import List
+from pydantic_ai.messages import ModelResponsePart
+from pydantic_ai.agent import AgentRunResult
+from pydantic_ai._run_context import RunContext
+from deepeval.test_case.llm_test_case import ToolCall
+
+
+# llm tools called
+def extract_tools_called_from_llm_response(
+    result: List[ModelResponsePart],
+) -> List[ToolCall]:
+    tool_calls = []
+
+    # Loop through each ModelResponsePart
+    for part in result:
+        # Look for parts with part_kind="tool-call"
+        if hasattr(part, "part_kind") and part.part_kind == "tool-call":
+            # Extract tool name and args from the ToolCallPart
+            tool_name = part.tool_name
+            input_parameters = (
+                part.args_as_dict() if hasattr(part, "args_as_dict") else None
+            )
+
+            # Create and append ToolCall object
+            tool_call = ToolCall(
+                name=tool_name, input_parameters=input_parameters
+            )
+            tool_calls.append(tool_call)
+
+    return tool_calls
+
+
+# TODO: llm tools called (reposne is present next message)
+def extract_tools_called(result: AgentRunResult) -> List[ToolCall]:
+    tool_calls = []
+
+    # Access the message history from the _state
+    message_history = result._state.message_history
+
+    # Scan through all messages in the history
+    for message in message_history:
+        # Check if this is a ModelResponse (kind="response")
+        if hasattr(message, "kind") and message.kind == "response":
+            # For ModelResponse messages, check each part
+            if hasattr(message, "parts"):
+                for part in message.parts:
+                    # Look for parts with part_kind="tool-call"
+                    if (
+                        hasattr(part, "part_kind")
+                        and part.part_kind == "tool-call"
+                    ):
+                        # Extract tool name and args from the ToolCallPart
+                        tool_name = part.tool_name
+                        input_parameters = (
+                            part.args_as_dict()
+                            if hasattr(part, "args_as_dict")
+                            else None
+                        )
+
+                        # Create and append ToolCall object
+                        tool_call = ToolCall(
+                            name=tool_name, input_parameters=input_parameters
+                        )
+                        tool_calls.append(tool_call)
+
+    return tool_calls
+
+
+def sanitize_run_context(value):
+    """
+    Recursively replace pydantic-ai RunContext instances with '<RunContext>'.
+
+    This avoids leaking internal context details into recorded function kwargs,
+    while keeping the original arguments intact for the actual function call.
+    """
+    if isinstance(value, RunContext):
+        return "<RunContext>"
+    if isinstance(value, dict):
+        return {k: sanitize_run_context(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        sanitized = [sanitize_run_context(v) for v in value]
+        return tuple(sanitized) if isinstance(value, tuple) else sanitized
+    if isinstance(value, set):
+        return {sanitize_run_context(v) for v in value}
+
+    return value

deepeval/metrics/conversational_g_eval/conversational_g_eval.py

@@ -316,6 +316,7 @@ class ConversationalGEval(BaseConversationalMetric):
         else:
             prompt = ConversationalGEvalTemplate.generate_evaluation_results(
                 evaluation_steps=self.number_evaluation_steps(),
+                test_case_content=test_case_content,
                 turns=[
                     convert_turn_to_dict(turn, self.evaluation_params)
                     for turn in test_case.turns

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -284,7 +284,7 @@ class PIILeakageMetric(BaseMetric):
                 no_privacy_count += 1
 
         score = no_privacy_count / number_of_verdicts
-        return 1 if self.strict_mode and score < 1 else score
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
         if self.error is not None: