deepeval 3.5.3__py3-none-any.whl → 3.5.4__py3-none-any.whl

deepeval/integrations/pydantic_ai/utils.py

@@ -1,8 +1,29 @@
-from typing import List
-from pydantic_ai.messages import ModelResponsePart
+from time import perf_counter
+from contextlib import asynccontextmanager
+import inspect
+import functools
+from typing import Any, Callable, List, Optional
+
+from pydantic_ai.models import Model
 from pydantic_ai.agent import AgentRunResult
 from pydantic_ai._run_context import RunContext
+from pydantic_ai.messages import (
+    ModelRequest,
+    ModelResponse,
+    ModelResponsePart,
+    SystemPromptPart,
+    TextPart,
+    ToolCallPart,
+    ToolReturnPart,
+    UserPromptPart,
+)
+
+from deepeval.prompt import Prompt
+from deepeval.tracing.tracing import Observer
+from deepeval.metrics.base_metric import BaseMetric
 from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import current_trace_context, current_span_context
+from deepeval.tracing.types import AgentSpan, LlmOutput, LlmSpan, LlmToolCall
 
 
 # llm tools called
@@ -84,3 +105,219 @@ def sanitize_run_context(value):
         return {sanitize_run_context(v) for v in value}
 
     return value
+
+
+def patch_llm_model(
+    model: Model,
+    llm_metric_collection: Optional[str] = None,
+    llm_metrics: Optional[List[BaseMetric]] = None,
+    llm_prompt: Optional[Prompt] = None,
+):
+    original_func = model.request
+    sig = inspect.signature(original_func)
+
+    try:
+        model_name = model.model_name
+    except Exception:
+        model_name = "unknown"
+
+    @functools.wraps(original_func)
+    async def wrapper(*args, **kwargs):
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        request = bound.arguments.get("messages", [])
+
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            observe_kwargs={"model": model_name},
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+        ) as observer:
+            result = await original_func(*args, **kwargs)
+            observer.update_span_properties = (
+                lambda llm_span: set_llm_span_attributes(
+                    llm_span, request, result, llm_prompt
+                )
+            )
+            observer.result = result
+            return result
+
+    model.request = wrapper
+
+    stream_original_func = model.request_stream
+    stream_sig = inspect.signature(stream_original_func)
+
+    @asynccontextmanager
+    async def stream_wrapper(*args, **kwargs):
+        bound = stream_sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        request = bound.arguments.get("messages", [])
+
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            observe_kwargs={"model": model_name},
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+        ) as observer:
+            llm_span: LlmSpan = current_span_context.get()
+            async with stream_original_func(
+                *args, **kwargs
+            ) as streamed_response:
+                try:
+                    yield streamed_response
+                    if not llm_span.token_intervals:
+                        llm_span.token_intervals = {perf_counter(): "NA"}
+                    else:
+                        llm_span.token_intervals[perf_counter()] = "NA"
+                finally:
+                    try:
+                        result = streamed_response.get()
+                        observer.update_span_properties = (
+                            lambda llm_span: set_llm_span_attributes(
+                                llm_span, request, result, llm_prompt
+                            )
+                        )
+                        observer.result = result
+                    except Exception:
+                        pass
+
+    model.request_stream = stream_wrapper
+
+
+def create_patched_tool(
+    func: Callable,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
+    import asyncio
+
+    original_func = func
+
+    is_async = asyncio.iscoroutinefunction(original_func)
+
+    if is_async:
+
+        @functools.wraps(original_func)
+        async def async_wrapper(*args, **kwargs):
+            sanitized_args = sanitize_run_context(args)
+            sanitized_kwargs = sanitize_run_context(kwargs)
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": sanitized_args, **sanitized_kwargs},
+            ) as observer:
+                result = await original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return async_wrapper
+    else:
+
+        @functools.wraps(original_func)
+        def sync_wrapper(*args, **kwargs):
+            sanitized_args = sanitize_run_context(args)
+            sanitized_kwargs = sanitize_run_context(kwargs)
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": sanitized_args, **sanitized_kwargs},
+            ) as observer:
+                result = original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return sync_wrapper
+
+
+def update_trace_context(
+    trace_name: Optional[str] = None,
+    trace_tags: Optional[List[str]] = None,
+    trace_metadata: Optional[dict] = None,
+    trace_thread_id: Optional[str] = None,
+    trace_user_id: Optional[str] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+    trace_input: Optional[Any] = None,
+    trace_output: Optional[Any] = None,
+):
+
+    current_trace = current_trace_context.get()
+
+    if trace_name:
+        current_trace.name = trace_name
+    if trace_tags:
+        current_trace.tags = trace_tags
+    if trace_metadata:
+        current_trace.metadata = trace_metadata
+    if trace_thread_id:
+        current_trace.thread_id = trace_thread_id
+    if trace_user_id:
+        current_trace.user_id = trace_user_id
+    if trace_metric_collection:
+        current_trace.metric_collection = trace_metric_collection
+    if trace_metrics:
+        current_trace.metrics = trace_metrics
+    if trace_input:
+        current_trace.input = trace_input
+    if trace_output:
+        current_trace.output = trace_output
+
+
+def set_llm_span_attributes(
+    llm_span: LlmSpan,
+    requests: List[ModelRequest],
+    result: ModelResponse,
+    llm_prompt: Optional[Prompt] = None,
+):
+    llm_span.prompt = llm_prompt
+
+    input = []
+    for request in requests:
+        for part in request.parts:
+            if isinstance(part, SystemPromptPart):
+                input.append({"role": "System", "content": part.content})
+            elif isinstance(part, UserPromptPart):
+                input.append({"role": "User", "content": part.content})
+            elif isinstance(part, ToolCallPart):
+                input.append(
+                    {
+                        "role": "Tool Call",
+                        "name": part.tool_name,
+                        "content": part.args_as_json_str(),
+                    }
+                )
+            elif isinstance(part, ToolReturnPart):
+                input.append(
+                    {
+                        "role": "Tool Return",
+                        "name": part.tool_name,
+                        "content": part.model_response_str(),
+                    }
+                )
+    llm_span.input = input
+
+    content = ""
+    tool_calls = []
+    for part in result.parts:
+        if isinstance(part, TextPart):
+            content += part.content + "\n"
+        elif isinstance(part, ToolCallPart):
+            tool_calls.append(
+                LlmToolCall(name=part.tool_name, args=part.args_as_dict())
+            )
+    llm_span.output = LlmOutput(
+        role="Assistant", content=content, tool_calls=tool_calls
+    )
+    llm_span.tools_called = extract_tools_called_from_llm_response(result.parts)
+
+
+def set_agent_span_attributes(agent_span: AgentSpan, result: AgentRunResult):
+    agent_span.tools_called = extract_tools_called(result)

deepeval/metrics/mcp_use_metric/mcp_use_metric.py

@@ -283,8 +283,9 @@ class MCPUseMetric(BaseMetric):
         mcp_resources_called: List[MCPResourceCall],
         mcp_prompts_called: List[MCPPromptCall],
     ) -> tuple[str, str]:
+        available_primitives = "MCP Primitives Available: \n"
         for mcp_server in mcp_servers:
-            available_primitives = f"MCP Server {mcp_server.server_name}\n"
+            available_primitives += f"MCP Server {mcp_server.server_name}\n"
             available_primitives += (
                 (
                     "\nAvailable Tools:\n[\n"

deepeval/metrics/non_advice/non_advice.py

@@ -43,7 +43,7 @@ class NonAdviceMetric(BaseMetric):
                 "or ['financial', 'medical'] for multiple types."
             )
 
-        self.threshold = 0 if strict_mode else threshold
+        self.threshold = 1 if strict_mode else threshold
         self.advice_types = advice_types
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
@@ -293,7 +293,7 @@ class NonAdviceMetric(BaseMetric):
                 appropriate_advice_count += 1
 
         score = appropriate_advice_count / number_of_verdicts
-        return 1 if self.strict_mode and score < 1 else score
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -35,7 +35,7 @@ class PIILeakageMetric(BaseMetric):
         verbose_mode: bool = False,
         evaluation_template: Type[PIILeakageTemplate] = PIILeakageTemplate,
     ):
-        self.threshold = 0 if strict_mode else threshold
+        self.threshold = 1 if strict_mode else threshold
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.include_reason = include_reason

{deepeval-3.5.3.dist-info → deepeval-3.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.3
+Version: 3.5.4
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

{deepeval-3.5.3.dist-info → deepeval-3.5.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deepeval/__init__.py,sha256=6fsb813LD_jNhqR-xZnSdE5E-KsBbC3tc4oIg5ZMgTw,2115
-deepeval/_version.py,sha256=FauBIJ2kq1CmkcSxNhoO0BZN8gc3azHQFgdBkDjya18,27
+deepeval/_version.py,sha256=Vy_DqdUIdzt42W7BKglfMO5ghp2Wa6OV5Tatx__sA2U,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -179,10 +179,11 @@ deepeval/integrations/llama_index/__init__.py,sha256=zBwUFQXDp6QFtp1cfANy8ucV08r
 deepeval/integrations/llama_index/agent/patched.py,sha256=4JbH0WQmt4lct7xxIH0phj8_Y-V35dgVv7DEDXK0jZI,2149
 deepeval/integrations/llama_index/handler.py,sha256=eqI1n8E4MsvfKoFs5Zrm9IdCR7g9eBgNedISs7UkU_I,8947
 deepeval/integrations/llama_index/utils.py,sha256=mxW71-3PjvBvJpLIU0kNWuTzCidy5l_-roLt8ZyWYA0,2599
-deepeval/integrations/pydantic_ai/__init__.py,sha256=36fBKBLRo1y5jFlj0Y4xhDJsiq4ZnqtmFO32R90Azo4,96
+deepeval/integrations/pydantic_ai/__init__.py,sha256=0-GZpWgCnFI-fVHI-3DosWQK85rk6CoRRhl4AiytBAw,258
+deepeval/integrations/pydantic_ai/agent.py,sha256=HxfeTLsdWGgRMy00ymgXdE6dcFDmFBsdgfl9BbvyJns,12311
 deepeval/integrations/pydantic_ai/otel.py,sha256=2DpO3RapdztXPlT9BWhQfF4dJDMyp2X7YvuplJ0SwC8,1661
-deepeval/integrations/pydantic_ai/patcher.py,sha256=C8CpY6UTO9oaai36l5C_GMT0Lqx9UVoJQYRPF0u0tMc,13127
-deepeval/integrations/pydantic_ai/utils.py,sha256=0BT3v1heuAnfhd9_FSp4XL8818MGdaI6oY2sTQtotfs,3211
+deepeval/integrations/pydantic_ai/patcher.py,sha256=yy4SZRmRhgYxh6qGVWWf8DnSMCDA9GLkFw1HbPToQ1w,17696
+deepeval/integrations/pydantic_ai/utils.py,sha256=734e9un-fn5V7MueAmVsXh304qgumv_fdcmdOC4HrJw,10998
 deepeval/key_handler.py,sha256=damdQEBLGy4IVk5DR5-E3blIZdLbcMtyeGAFn_4_SG4,6505
 deepeval/metrics/__init__.py,sha256=nvO0Wv2JROjK1I9MDNIFUJlrRAZI2C0xbGYSBZK5q4g,4013
 deepeval/metrics/answer_relevancy/__init__.py,sha256=WbZUpoSg2GQoqJ4VIRirVVQ1JDx5xwT-RskwqNKfWGM,46
@@ -262,7 +263,7 @@ deepeval/metrics/mcp/multi_turn_mcp_use_metric.py,sha256=XegYpPVH0qR5lKqQUjMg8dx
 deepeval/metrics/mcp/schema.py,sha256=e9_bFfI8uHeejaePu-YIX8qpAax1noPaKhpiD_NYlgg,310
 deepeval/metrics/mcp/template.py,sha256=iL1V9W40piCAlstk_qYOTHAy2aymqbMmujHempUk25s,5482
 deepeval/metrics/mcp_use_metric/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/metrics/mcp_use_metric/mcp_use_metric.py,sha256=71WhX-GPD8zdkUhcdSfB81aGORojK6TODpx-Fw3NOac,14017
+deepeval/metrics/mcp_use_metric/mcp_use_metric.py,sha256=2Q0fhNfLEoUCiz-NaG8UKNthdAywgexT2mqBXxipTPk,14080
 deepeval/metrics/mcp_use_metric/schema.py,sha256=GAWacRNl0i7ir_AE_f1_OBEj0Q5xhcIwEytmTFpiwyw,169
 deepeval/metrics/mcp_use_metric/template.py,sha256=iUttypDHYPc_QPB8hvuPhmSodJMhqUpau3P53thnE_Y,5625
 deepeval/metrics/misuse/__init__.py,sha256=TqtaJf0zzFceJtb3BSTT_hTA6OzgnF3Y-XuWjR2bgVs,37
@@ -318,11 +319,11 @@ deepeval/metrics/multimodal_metrics/text_to_image/schema.py,sha256=ygt_RGnVlYh__
 deepeval/metrics/multimodal_metrics/text_to_image/template.py,sha256=WSXXI0Tee1wE7FPyQJwHYXuqHaevYz9T04ns1P85Qec,2568
 deepeval/metrics/multimodal_metrics/text_to_image/text_to_image.py,sha256=20wzu09kQhNMxjp7oJ_sMfWXOWNhuj0kkyEed9lwS2g,11160
 deepeval/metrics/non_advice/__init__.py,sha256=GP55jVADpkODABIjzK0JX1MKpFNZ0bM7Q2Jczlc4BSU,40
-deepeval/metrics/non_advice/non_advice.py,sha256=DebtVrzAdwvoJ1eWovC6ahjY1qdsxUCYl2lQYkS3vI4,11234
+deepeval/metrics/non_advice/non_advice.py,sha256=-pyy1uXfCn0Yuo-JnM4H9lakKzYyzRlIWQ3cA5SwpuM,11247
 deepeval/metrics/non_advice/schema.py,sha256=bODTV8jfjIYTwnYRHz32p47tqdXkTRqLXj_s5ZUxYAQ,299
 deepeval/metrics/non_advice/template.py,sha256=KiRoU_Re3JFHylKZ1O8hztZ3yEQf3vW_HWwHxQjDb6o,2864
 deepeval/metrics/pii_leakage/__init__.py,sha256=tBc9OGp4gmgoYz6FA3ipr48fpsCMvq6WtlwOjMqhCD0,42
-deepeval/metrics/pii_leakage/pii_leakage.py,sha256=sZPCjlegbs_djexoOGg5WEiYUgEYZQFPPYlRDk_FRUc,10851
+deepeval/metrics/pii_leakage/pii_leakage.py,sha256=EIQMS_hOiYhEW5x4nYJwS6AhWl9jhN261atVoWZI3f4,10851
 deepeval/metrics/pii_leakage/schema.py,sha256=Jk9jdf4HAa76J237mnosWOCV71pBBNdLfaVhf-4dKEg,313
 deepeval/metrics/pii_leakage/template.py,sha256=DEW21CyR2lEI1y2C_fXgZnGJlYw0fvnB-LF-HEKZnqo,2418
 deepeval/metrics/prompt_alignment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -460,8 +461,8 @@ deepeval/tracing/tracing.py,sha256=vOVFdN6fVMW53XhyqTZSfp4vI7DCqRez4TKNhdhr-sg,4
 deepeval/tracing/types.py,sha256=l_utWKerNlE5H3mOKpeUJLsvpP3cMyjH7HRANNgTmSQ,5306
 deepeval/tracing/utils.py,sha256=w_kdhuyBCygllnbqLpDdKJqpJo42t3ZMlGhNicV2A8c,6467
 deepeval/utils.py,sha256=r8tV_NYJSi6ib-oQw6cLw3L7ZSe4KIJVJc1ng6-kDX4,17179
-deepeval-3.5.3.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.5.3.dist-info/METADATA,sha256=WDYyYAc2YdYGWufFtlvsEFsV___80J_xPPCMkvlwau8,18682
-deepeval-3.5.3.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.5.3.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
-deepeval-3.5.3.dist-info/RECORD,,
+deepeval-3.5.4.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.5.4.dist-info/METADATA,sha256=fJ15yXxlzKTfOsoW5z7uxIJ4Qx6X-UTpKj7pabi5Tv8,18682
+deepeval-3.5.4.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.5.4.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
+deepeval-3.5.4.dist-info/RECORD,,