deepeval 3.4.9__py3-none-any.whl → 3.5.0__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.4.9"
+__version__: str = "3.5.0"

deepeval/benchmarks/drop/drop.py

@@ -1,6 +1,5 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
-from typing import Union
 
 from deepeval.dataset import Golden
 from deepeval.benchmarks.base_benchmark import (
@@ -50,7 +49,7 @@ class DROP(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

deepeval/benchmarks/hellaswag/hellaswag.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -51,7 +51,7 @@ class HellaSwag(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

deepeval/benchmarks/logi_qa/logi_qa.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 import requests
 import json
@@ -52,7 +52,7 @@ class LogiQA(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

deepeval/benchmarks/math_qa/math_qa.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -50,7 +50,7 @@ class MathQA(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

deepeval/benchmarks/mmlu/mmlu.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -49,7 +49,7 @@ class MMLU(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

deepeval/benchmarks/truthful_qa/truthful_qa.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -59,7 +59,7 @@ class TruthfulQA(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

deepeval/confident/api.py

@@ -10,6 +10,7 @@ from tenacity import (
     retry_if_exception_type,
     RetryCallState,
 )
+from pydantic import SecretStr
 
 import deepeval
 from deepeval.key_handler import KEY_FILE_HANDLER, KeyValues

deepeval/integrations/pydantic_ai/__init__.py

@@ -1,5 +1,3 @@
-from .agent import PydanticAIAgent as Agent
-from .setup import instrument_pydantic_ai
+from .patcher import instrument as instrument_pydantic_ai
 
-
-__all__ = ["Agent", "instrument_pydantic_ai"]
+__all__ = ["instrument_pydantic_ai"]

deepeval/integrations/pydantic_ai/{setup.py → otel.py}

@@ -31,14 +31,6 @@ def instrument_pydantic_ai(api_key: Optional[str] = None):
     with capture_tracing_integration("pydantic_ai"):
         is_opentelemetry_available()
 
-        if api_key:
-            deepeval.login(api_key)
-
-        api_key = get_confident_api_key()
-
-        if not api_key:
-            raise ValueError("No api key provided.")
-
         # create a new tracer provider
         tracer_provider = TracerProvider()
         tracer_provider.add_span_processor(

deepeval/integrations/pydantic_ai/patcher.py

@@ -0,0 +1,376 @@
+import functools
+import deepeval
+from deepeval.tracing.types import LlmOutput, LlmToolCall
+from pydantic_ai.agent import AgentRunResult
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import AgentSpan, LlmSpan
+from deepeval.tracing.tracing import Observer
+from typing import List, Callable, Optional, Any
+from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.metrics.base_metric import BaseMetric
+from deepeval.confident.api import get_confident_api_key
+from deepeval.integrations.pydantic_ai.otel import instrument_pydantic_ai
+from deepeval.telemetry import capture_tracing_integration
+from deepeval.prompt import Prompt
+
+try:
+    from pydantic_ai.agent import Agent
+    from pydantic_ai.models import Model
+    from pydantic_ai.messages import (
+        ModelResponse,
+        ModelRequest,
+        ModelResponsePart,
+        TextPart,
+        ToolCallPart,
+        SystemPromptPart,
+        ToolReturnPart,
+        UserPromptPart,
+    )
+
+    pydantic_ai_installed = True
+except:
+    pydantic_ai_installed = True
+
+
+def _patch_agent_tool_decorator():
+    original_tool = Agent.tool
+
+    @functools.wraps(original_tool)
+    def wrapper(
+        *args,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        **kwargs
+    ):
+        # Case 1: Direct decoration - @agent.tool
+        if args and callable(args[0]):
+            patched_func = _create_patched_tool(
+                args[0], metrics, metric_collection
+            )
+            new_args = (patched_func,) + args[1:]
+            return original_tool(*new_args, **kwargs)
+
+        # Case 2: Decoration with arguments - @agent.tool(metrics=..., metric_collection=...)
+        else:
+            # Return a decorator function that will receive the actual function
+            def decorator(func):
+                patched_func = _create_patched_tool(
+                    func, metrics, metric_collection
+                )
+                return original_tool(*args, **kwargs)(patched_func)
+
+            return decorator
+
+    Agent.tool = wrapper
+
+
+def _create_patched_tool(
+    func: Callable,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
+    import asyncio
+
+    original_func = func
+
+    is_async = asyncio.iscoroutinefunction(original_func)
+
+    if is_async:
+
+        @functools.wraps(original_func)
+        async def async_wrapper(*args, **kwargs):
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": args, **kwargs},
+            ) as observer:
+                result = await original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return async_wrapper
+    else:
+
+        @functools.wraps(original_func)
+        def sync_wrapper(*args, **kwargs):
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": args, **kwargs},
+            ) as observer:
+                result = original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return sync_wrapper
+
+
+def _patch_agent_init():
+    original_init = Agent.__init__
+
+    @functools.wraps(original_init)
+    def wrapper(
+        self,
+        *args,
+        llm_metric_collection: Optional[str] = None,
+        llm_metrics: Optional[List[BaseMetric]] = None,
+        llm_prompt: Optional[Prompt] = None,
+        agent_metric_collection: Optional[str] = None,
+        agent_metrics: Optional[List[BaseMetric]] = None,
+        **kwargs
+    ):
+        result = original_init(self, *args, **kwargs)
+        _patch_llm_model(
+            self._model, llm_metric_collection, llm_metrics, llm_prompt
+        )  # runtime patch of the model
+        _patch_agent_run(agent_metric_collection, agent_metrics)
+        return result
+
+    Agent.__init__ = wrapper
+
+
+def _patch_agent_run(
+    agent_metric_collection: Optional[str] = None,
+    agent_metrics: Optional[List[BaseMetric]] = None,
+):
+    original_run = Agent.run
+
+    @functools.wraps(original_run)
+    async def wrapper(
+        *args,
+        trace_metric_collection: Optional[str] = None,
+        trace_metrics: Optional[List[BaseMetric]] = None,
+        trace_name: Optional[str] = None,
+        trace_tags: Optional[List[str]] = None,
+        trace_metadata: Optional[dict] = None,
+        trace_thread_id: Optional[str] = None,
+        trace_user_id: Optional[str] = None,
+        **kwargs
+    ):
+        with Observer(
+            span_type="agent",
+            func_name="Agent",
+            function_kwargs={"input": args[1]},
+            metrics=agent_metrics,
+            metric_collection=agent_metric_collection,
+        ) as observer:
+            result = await original_run(*args, **kwargs)
+            observer.update_span_properties = (
+                lambda agent_span: set_agent_span_attributes(agent_span, result)
+            )
+            observer.result = result.output
+
+            _update_trace_context(
+                trace_name=trace_name,
+                trace_tags=trace_tags,
+                trace_metadata=trace_metadata,
+                trace_thread_id=trace_thread_id,
+                trace_user_id=trace_user_id,
+                trace_metric_collection=trace_metric_collection,
+                trace_metrics=trace_metrics,
+                trace_input=args[1],
+                trace_output=result.output,
+            )
+
+        return result
+
+    Agent.run = wrapper
+
+
+def _update_trace_context(
+    trace_name: Optional[str] = None,
+    trace_tags: Optional[List[str]] = None,
+    trace_metadata: Optional[dict] = None,
+    trace_thread_id: Optional[str] = None,
+    trace_user_id: Optional[str] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+    trace_input: Optional[Any] = None,
+    trace_output: Optional[Any] = None,
+):
+
+    current_trace = current_trace_context.get()
+    current_trace.name = trace_name
+    current_trace.tags = trace_tags
+    current_trace.metadata = trace_metadata
+    current_trace.thread_id = trace_thread_id
+    current_trace.user_id = trace_user_id
+    current_trace.metric_collection = trace_metric_collection
+    current_trace.metrics = trace_metrics
+    current_trace.input = trace_input
+    current_trace.output = trace_output
+
+
+def _patch_llm_model(
+    model: Model,
+    llm_metric_collection: Optional[str] = None,
+    llm_metrics: Optional[List[BaseMetric]] = None,
+    llm_prompt: Optional[Prompt] = None,
+):
+    original_func = model.request
+    try:
+        model_name = model.model_name
+    except Exception:
+        model_name = "unknown"
+
+    @functools.wraps(original_func)
+    async def wrapper(*args, **kwargs):
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            observe_kwargs={"model": model_name},
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+        ) as observer:
+            result = await original_func(*args, **kwargs)
+            request = kwargs.get("messages", [])
+            if not request:
+                request = args[0]
+            observer.update_span_properties = (
+                lambda llm_span: set_llm_span_attributes(
+                    llm_span, args[0], result, llm_prompt
+                )
+            )
+            observer.result = result
+        return result
+
+    model.request = wrapper
+
+
+def instrument(otel: Optional[bool] = False, api_key: Optional[str] = None):
+
+    if api_key:
+        deepeval.login(api_key)
+
+    api_key = get_confident_api_key()
+
+    if not api_key:
+        raise ValueError("No api key provided.")
+
+    if otel:
+        instrument_pydantic_ai(api_key)
+    else:
+        with capture_tracing_integration("pydantic_ai"):
+            _patch_agent_init()
+            _patch_agent_tool_decorator()
+
+
+def set_llm_span_attributes(
+    llm_span: LlmSpan,
+    requests: List[ModelRequest],
+    result: ModelResponse,
+    llm_prompt: Optional[Prompt] = None,
+):
+    llm_span.prompt = llm_prompt
+
+    input = []
+    for request in requests:
+        for part in request.parts:
+            if isinstance(part, SystemPromptPart):
+                input.append({"role": "System", "content": part.content})
+            elif isinstance(part, UserPromptPart):
+                input.append({"role": "User", "content": part.content})
+            elif isinstance(part, ToolCallPart):
+                input.append(
+                    {
+                        "role": "Tool Call",
+                        "name": part.tool_name,
+                        "content": part.args_as_json_str(),
+                    }
+                )
+            elif isinstance(part, ToolReturnPart):
+                input.append(
+                    {
+                        "role": "Tool Return",
+                        "name": part.tool_name,
+                        "content": part.model_response_str(),
+                    }
+                )
+    llm_span.input = input
+
+    content = ""
+    tool_calls = []
+    for part in result.parts:
+        if isinstance(part, TextPart):
+            content += part.content + "\n"
+        elif isinstance(part, ToolCallPart):
+            tool_calls.append(
+                LlmToolCall(name=part.tool_name, args=part.args_as_dict())
+            )
+    llm_span.output = LlmOutput(
+        role="Assistant", content=content, tool_calls=tool_calls
+    )
+    llm_span.tools_called = _extract_tools_called_from_llm_response(
+        result.parts
+    )
+
+
+def set_agent_span_attributes(agent_span: AgentSpan, result: AgentRunResult):
+    agent_span.tools_called = _extract_tools_called(result)
+
+
+# llm tools called
+def _extract_tools_called_from_llm_response(
+    result: List[ModelResponsePart],
+) -> List[ToolCall]:
+    tool_calls = []
+
+    # Loop through each ModelResponsePart
+    for part in result:
+        # Look for parts with part_kind="tool-call"
+        if hasattr(part, "part_kind") and part.part_kind == "tool-call":
+            # Extract tool name and args from the ToolCallPart
+            tool_name = part.tool_name
+            input_parameters = (
+                part.args_as_dict() if hasattr(part, "args_as_dict") else None
+            )
+
+            # Create and append ToolCall object
+            tool_call = ToolCall(
+                name=tool_name, input_parameters=input_parameters
+            )
+            tool_calls.append(tool_call)
+
+    return tool_calls
+
+
+# TODO: llm tools called (reposne is present next message)
+def _extract_tools_called(result: AgentRunResult) -> List[ToolCall]:
+    tool_calls = []
+
+    # Access the message history from the _state
+    message_history = result._state.message_history
+
+    # Scan through all messages in the history
+    for message in message_history:
+        # Check if this is a ModelResponse (kind="response")
+        if hasattr(message, "kind") and message.kind == "response":
+            # For ModelResponse messages, check each part
+            if hasattr(message, "parts"):
+                for part in message.parts:
+                    # Look for parts with part_kind="tool-call"
+                    if (
+                        hasattr(part, "part_kind")
+                        and part.part_kind == "tool-call"
+                    ):
+                        # Extract tool name and args from the ToolCallPart
+                        tool_name = part.tool_name
+                        input_parameters = (
+                            part.args_as_dict()
+                            if hasattr(part, "args_as_dict")
+                            else None
+                        )
+
+                        # Create and append ToolCall object
+                        tool_call = ToolCall(
+                            name=tool_name, input_parameters=input_parameters
+                        )
+                        tool_calls.append(tool_call)
+
+    return tool_calls

deepeval/models/llms/grok_model.py

@@ -30,7 +30,7 @@ model_pricing = {
     },
     "grok-3-fast": {
         "input": 0.60 / 1e6,
-        "output": 2.50 / 1e-6,
+        "output": 2.50 / 1e6,
     },
     "grok-3-mini-fast": {
         "input": 30 / 1e6,

deepeval/models/llms/kimi_model.py

@@ -30,7 +30,7 @@ model_pricing = {
     },
     "kimi-k2-0711-preview": {
         "input": 0.60 / 1e6,
-        "output": 2.50 / 1e-6,
+        "output": 2.50 / 1e6,
     },
     "kimi-thinking-preview": {
         "input": 30 / 1e6,

deepeval/tracing/perf_epoch_bridge.py

@@ -15,12 +15,12 @@ Usage:
 
 from __future__ import annotations
 import time
-from typing import Final
+from typing import Final, Union
 
 # Module globals are initialised exactly once.
-_anchor_perf_ns: int | None = None
-_anchor_wall_ns: int | None = None
-_offset_ns: int | None = None
+_anchor_perf_ns: Union[int, None] = None
+_anchor_wall_ns: Union[int, None] = None
+_offset_ns: Union[int, None] = None
 
 
 def init_clock_bridge() -> None:

deepeval/utils.py

@@ -516,7 +516,7 @@ def remove_pbars(
 
 
 def read_env_int(
-    name: str, default: int, *, min_value: int | None = None
+    name: str, default: int, *, min_value: Union[int, None] = None
 ) -> int:
     """Read an integer from an environment variable with safe fallback.
 
@@ -545,7 +545,7 @@ def read_env_int(
 
 
 def read_env_float(
-    name: str, default: float, *, min_value: float | None = None
+    name: str, default: float, *, min_value: Union[float, None] = None
 ) -> float:
     """Read a float from an environment variable with safe fallback.
 

{deepeval-3.4.9.dist-info → deepeval-3.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.4.9
+Version: 3.5.0
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -189,16 +189,6 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 ```
 pip install -U deepeval
 ```
-### Environment variables (.env / .env.local)
-
-DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
-**Precedence:** process env -> `.env.local` -> `.env`.
-Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
-
-```bash
-cp .env.example .env.local
-# then edit .env.local (ignored by git)
-```
 
 ## Create an account (highly recommended)
 
@@ -391,9 +381,20 @@ evaluate(dataset, [answer_relevancy_metric])
 dataset.evaluate([answer_relevancy_metric])
 ```
 
-# LLM Evaluation With Confident AI
+## A Note on Env Variables (.env / .env.local)
+
+DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
+**Precedence:** process env -> `.env.local` -> `.env`.
+Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+```
+
+# DeepEval With Confident AI
 
-The correct LLM evaluation lifecycle is only achievable with [the DeepEval platform](https://confident-ai.com?utm_source=Github). It allows you to:
+DeepEval's cloud platform, [Confident AI](https://confident-ai.com?utm_source=Github), allows you to:
 
 1. Curate/annotate evaluation datasets on the cloud
 2. Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best

{deepeval-3.4.9.dist-info → deepeval-3.5.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deepeval/__init__.py,sha256=6fsb813LD_jNhqR-xZnSdE5E-KsBbC3tc4oIg5ZMgTw,2115
-deepeval/_version.py,sha256=C51bPZRI51sTy21Ve1RwtpAOCNmb0BUammiT884H3Ow,27
+deepeval/_version.py,sha256=xgoMNdDXsY3c4GfV1_DVK-xGdMOp5KCDaKln5j0PJdY,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -77,7 +77,7 @@ deepeval/benchmarks/bool_q/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 deepeval/benchmarks/bool_q/bool_q.py,sha256=wJM4-wSybT8EwgDJVB4p3QYXGNzLD3tdrpGE1cNEz_E,5507
 deepeval/benchmarks/bool_q/template.py,sha256=pgNj4RR6-4VJDDySwnKt-MpghBCjVlZ7fPKY6PltllQ,4055
 deepeval/benchmarks/drop/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/benchmarks/drop/drop.py,sha256=Qo74clH8ERf1Zutcz76wrTbR-fbR2qHdsZP6fJ1W-LQ,12054
+deepeval/benchmarks/drop/drop.py,sha256=rGcqd79-IfQ2tvPuAL6wrON4R0hBiVGBy1OtDRmertE,12042
 deepeval/benchmarks/drop/task.py,sha256=RV7DEXF192IOsY-yIVdlGb_y-A_sS5APPn8PGOPn5yU,17950
 deepeval/benchmarks/drop/template.py,sha256=1P0mx_71Bxr9juIA8nGpVRIrP8NSoDILkIicjWvqE94,1376
 deepeval/benchmarks/equity_med_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -88,7 +88,7 @@ deepeval/benchmarks/gsm8k/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 deepeval/benchmarks/gsm8k/gsm8k.py,sha256=LyJQBskKuqp013LLO3539RJiRXqCmlGl12BPXvQ8P88,6974
 deepeval/benchmarks/gsm8k/template.py,sha256=3F7DwQwhJwKxtlbaO6TNvBBRaDEUBEp58JwirSjxtR0,1626
 deepeval/benchmarks/hellaswag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/benchmarks/hellaswag/hellaswag.py,sha256=ESX4A8WRxSeErPC34QuMM6WVEeDeyiCmNq10twZJxgw,11959
+deepeval/benchmarks/hellaswag/hellaswag.py,sha256=_3felzBwQUhhRXk4D9NbcY8dme_qUQcwUjKGw9OtDJg,11972
 deepeval/benchmarks/hellaswag/task.py,sha256=LfO8T6bpNiwdM8VdubKrup7qje3-rHgu69iB6Sdsc6I,7323
 deepeval/benchmarks/hellaswag/template.py,sha256=TcCu25hkl89qbRwcEyRVGTGp7DU_5Eph754W2znk5QY,1279
 deepeval/benchmarks/human_eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -102,15 +102,15 @@ deepeval/benchmarks/lambada/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 deepeval/benchmarks/lambada/lambada.py,sha256=FExZLpDBgQfYe9o-MBS0LEy0-i4jHGeFHo8XCbMW_io,5556
 deepeval/benchmarks/lambada/template.py,sha256=mSn0Elvp34wTnvaAm3IENz0mfGSNM_iRx50hIouk4t0,3776
 deepeval/benchmarks/logi_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/benchmarks/logi_qa/logi_qa.py,sha256=zN7KTa77d1WkM_m9V-Z0AfHBLFZ70s89yMaQ3_CNV84,11130
+deepeval/benchmarks/logi_qa/logi_qa.py,sha256=VNZGASigEDlJjzwGZtWG3OUs3v3P733GD84-h3TaxjU,11143
 deepeval/benchmarks/logi_qa/task.py,sha256=pVMLVHPyDFSyoIsnckBNRDt8FK0J317PiGT-0dpr7rs,350
 deepeval/benchmarks/logi_qa/template.py,sha256=EddGd2s3u2bPejogTcM50SDS7ynHnMhHaKuqQjjZoLk,4354
 deepeval/benchmarks/math_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/benchmarks/math_qa/math_qa.py,sha256=yRlVkUb_L4fPJIvaDo-5EB0XeVId9CxrIZUX7wLQJh4,10638
+deepeval/benchmarks/math_qa/math_qa.py,sha256=_eP-yocJom9r91qmAUBbIH4hrWazEHLV2lDKu0yMfEI,10651
 deepeval/benchmarks/math_qa/task.py,sha256=3q_jlK5kIl5Zs0mQwuzxyvmPP6ncLZwszn7gtl1GfZs,192
 deepeval/benchmarks/math_qa/template.py,sha256=pC3PB2GGU5TQ81I7E76RJh0xlu7xiF6d4SK3T_Nksh8,4468
 deepeval/benchmarks/mmlu/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/benchmarks/mmlu/mmlu.py,sha256=KEiiRSfQQQSX51dcnrWPkQcud1WvcKe1v0T82P1NowY,11418
+deepeval/benchmarks/mmlu/mmlu.py,sha256=flg3tb052DVo7wnfAHkW9n07tEEhHrkT2C0d5-UMBoQ,11431
 deepeval/benchmarks/mmlu/task.py,sha256=HnhnuD4Xjur9GlrBtswaR7ZPouGx4NTgbcFZu_oIzXw,2580
 deepeval/benchmarks/mmlu/template.py,sha256=MsdcrZWVkyZpEw--Kj6W7vjOJgig-ABiz9B3WtZz1MQ,1303
 deepeval/benchmarks/modes/__init__.py,sha256=IGhZp0-nmvVsZWBnTuBvKhdGiy4TJZShFSjYAeBZdbo,135
@@ -125,7 +125,7 @@ deepeval/benchmarks/truthful_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 deepeval/benchmarks/truthful_qa/mode.py,sha256=66aXCTXGTbAprA33M3TT15OhpJAqxLPDzJuShKxiFwY,84
 deepeval/benchmarks/truthful_qa/task.py,sha256=PmfPbqINd9wizq8Tpk8pwms9TersoGlMGBqxpTmZhcc,1360
 deepeval/benchmarks/truthful_qa/template.py,sha256=5y6mfJm9AXnQL7xwrfsZjH080GwO1kd_1GdTzDCoYgo,4465
-deepeval/benchmarks/truthful_qa/truthful_qa.py,sha256=j8RzrAwgKVt2oWNf1bz9cPIk6b-I7WsjiPElJlQ21Xs,13768
+deepeval/benchmarks/truthful_qa/truthful_qa.py,sha256=2r-xcFnzSSJds3ZGxYogBzjFFrCfJxYpXzKrpE8cC_c,13781
 deepeval/benchmarks/utils.py,sha256=NHImqH22mv108_CKM7ajTpu4hOeUhr5xPicbf0i2qGk,287
 deepeval/benchmarks/winogrande/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/benchmarks/winogrande/template.py,sha256=tDwH8NpNF9x7FbDmQw45XaW1LNqGBV6zP5pwV1uOlwM,2089
@@ -138,7 +138,7 @@ deepeval/cli/test.py,sha256=kSIFMRTAfVzBJ4OitwvT829-ylV7UzPMP57P2DePS-Q,5482
 deepeval/cli/types.py,sha256=_7KdthstHNc-JKCWrfpDQCf_j8h9PMxh0qJCHmVXJr0,310
 deepeval/cli/utils.py,sha256=F4-yuONzk4ojDoSLjI9RYERB7HOD412iZ2lNlSCq4wk,5601
 deepeval/confident/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-deepeval/confident/api.py,sha256=tUv5P5nSdY-RHjH1FQ9AlJ36Ut8QAxCHWeARks64-Vo,8287
+deepeval/confident/api.py,sha256=-2i3IBLtj5bUIImwOF6ltGVR3ZyViIbIC38XxwWvf54,8318
 deepeval/confident/types.py,sha256=-slFhDof_1maMgpLxqDRZv6kz6ZVY2hP_0uj_aveJKU,533
 deepeval/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/config/settings.py,sha256=e7sk6_3I14hG457e75DoJd9Ojo3rOkpBZzsMYlj4gKQ,18139
@@ -179,9 +179,9 @@ deepeval/integrations/llama_index/__init__.py,sha256=zBwUFQXDp6QFtp1cfANy8ucV08r
 deepeval/integrations/llama_index/agent/patched.py,sha256=4JbH0WQmt4lct7xxIH0phj8_Y-V35dgVv7DEDXK0jZI,2149
 deepeval/integrations/llama_index/handler.py,sha256=eqI1n8E4MsvfKoFs5Zrm9IdCR7g9eBgNedISs7UkU_I,8947
 deepeval/integrations/llama_index/utils.py,sha256=mxW71-3PjvBvJpLIU0kNWuTzCidy5l_-roLt8ZyWYA0,2599
-deepeval/integrations/pydantic_ai/__init__.py,sha256=04qKTNftGVzso3n7F1hvKxXI-NG3Enx8TP-EdpJykUc,134
-deepeval/integrations/pydantic_ai/agent.py,sha256=6PB_-zWaivQR65xi2F_WqBNtOcUwRpvI8wyCfpGhJJQ,13549
-deepeval/integrations/pydantic_ai/setup.py,sha256=-G15PEHZpLUsGNa7yP1IP2t3_6aKt5ac38930VnUYQY,1839
+deepeval/integrations/pydantic_ai/__init__.py,sha256=36fBKBLRo1y5jFlj0Y4xhDJsiq4ZnqtmFO32R90Azo4,96
+deepeval/integrations/pydantic_ai/otel.py,sha256=2DpO3RapdztXPlT9BWhQfF4dJDMyp2X7YvuplJ0SwC8,1661
+deepeval/integrations/pydantic_ai/patcher.py,sha256=wszU2YROZAQovyz1ZNRvTtsuJ5By_x4SF6yjtmItcNk,12210
 deepeval/key_handler.py,sha256=damdQEBLGy4IVk5DR5-E3blIZdLbcMtyeGAFn_4_SG4,6505
 deepeval/metrics/__init__.py,sha256=xofaK_bJq0QCSerSWYjHYRXXch9YQwZHxIfVAv1G7fo,4012
 deepeval/metrics/answer_relevancy/__init__.py,sha256=WbZUpoSg2GQoqJ4VIRirVVQ1JDx5xwT-RskwqNKfWGM,46
@@ -373,8 +373,8 @@ deepeval/models/llms/anthropic_model.py,sha256=T55-jKRbM3_B3Db9M3ruklm2cVVU1JDGA
 deepeval/models/llms/azure_model.py,sha256=MG6sVGUgIy2RURwFWvRP7O_RF6QAg2dpqXIJhIsgY60,10994
 deepeval/models/llms/deepseek_model.py,sha256=mz0U0uqazAVr8vv8SF74GRTr4ZEVc3Q1v9o5TVbmz_8,5440
 deepeval/models/llms/gemini_model.py,sha256=QXf9mjopfWwJxpm0gbkXo6op_Wtu1GaIt1BfzS3OU8Q,8174
-deepeval/models/llms/grok_model.py,sha256=S_ROLB3NWWyyY5tzezln1xifjWRgwdVGOraIpQPjF2Q,5987
-deepeval/models/llms/kimi_model.py,sha256=1axzi_DgtVDno8tb9U8FrRbqKK6LSeL7sQEuQqEsM5A,6727
+deepeval/models/llms/grok_model.py,sha256=zPBmPnNCRGrtg_709gFv5A4iz7WilTmDpAyOpjXTa_M,5986
+deepeval/models/llms/kimi_model.py,sha256=ZcvEwWgnv1dtmbq7LgMQJAjpkjxZr-l5eBi9KGqRbb0,6726
 deepeval/models/llms/litellm_model.py,sha256=iu4-_JCpd9LdEa-eCWseD2iLTA-r7OSgYGWQ0IxB4eA,11527
 deepeval/models/llms/local_model.py,sha256=PeF6ofMR8jBMTLzkCkgmkBJix9kHbWV5vTKGx8nehFs,3605
 deepeval/models/llms/ollama_model.py,sha256=foL6sMza37Z0HH8qPStyIr1g-xEaD6Ce53L2C8Er-P8,3055
@@ -454,13 +454,13 @@ deepeval/tracing/otel/__init__.py,sha256=HQsaF5yLPwyW5qg8AOV81_nG_7pFHnatOTHi9Wx
 deepeval/tracing/otel/exporter.py,sha256=dXQd834zm5rm1ss9pWkBBlk-JSdtiw7aFLso2hM53XY,26372
 deepeval/tracing/otel/utils.py,sha256=g8yAzhqbPh1fOKCWkfNekC6AVotLfu1SUcfNMo6zii8,9786
 deepeval/tracing/patchers.py,sha256=DAPNkhrDtoeyJIVeQDUMhTz-xGcXu00eqjQZmov8FiU,3096
-deepeval/tracing/perf_epoch_bridge.py,sha256=9g5bdQMKrhXtlS5LgC-oDNE7foe307DE07NqkusW6GU,1800
+deepeval/tracing/perf_epoch_bridge.py,sha256=iyAPddB6Op7NpMtPHJ29lDm53Btz9yLaN6xSCfTRQm4,1825
 deepeval/tracing/tracing.py,sha256=StvwFEG3MG67n7PBEyDDycdj0myMbP3LMB_FBhaZH-Y,38741
 deepeval/tracing/types.py,sha256=3w5HEI6y4zuzVr8xGEEzDviLZCX_s_pK85qbwnyf1aY,5196
 deepeval/tracing/utils.py,sha256=eTEickbDvRiOu1twNolh4sHnjZF49vqdLgI74BudeTw,6357
-deepeval/utils.py,sha256=SSCk7LLZQG-phJG8cuVPX-C21ZDCa_D2jWw0uJb6Ov8,16845
-deepeval-3.4.9.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.4.9.dist-info/METADATA,sha256=_iUKm_ZCpwntM0u23zLZBQeTjPV4ogVwr5zX-K4BpN4,18732
-deepeval-3.4.9.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.4.9.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
-deepeval-3.4.9.dist-info/RECORD,,
+deepeval/utils.py,sha256=EimWDwI1pKCE8vl6kuTnGbGT6ep9zHL5sZ0o-gj49XI,16857
+deepeval-3.5.0.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.5.0.dist-info/METADATA,sha256=KBAB5m11q4GAhVwCJBmXZDtaYtKoAO3sQ0vg-ajFRLg,18682
+deepeval-3.5.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.5.0.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
+deepeval-3.5.0.dist-info/RECORD,,

deepeval/integrations/pydantic_ai/agent.py

@@ -1,364 +0,0 @@
-from deepeval.telemetry import capture_tracing_integration
-from deepeval.metrics import BaseMetric
-from typing import List, Optional
-import functools
-import inspect
-import json
-from deepeval.test_case import LLMTestCase
-from deepeval.tracing.types import TestCaseMetricPair
-from deepeval.tracing.tracing import trace_manager
-from deepeval.tracing.otel.utils import parse_string, parse_list_of_strings
-from opentelemetry import trace
-
-try:
-    from opentelemetry.trace import NoOpTracer
-
-    opentelemetry_installed = True
-except:
-    opentelemetry_installed = False
-
-
-def is_opentelemetry_available():
-    if not opentelemetry_installed:
-        raise ImportError(
-            "OpenTelemetry SDK is not available. Please install it with `pip install opentelemetry-sdk`."
-        )
-    return True
-
-
-try:
-    from pydantic_ai.agent import Agent
-    from pydantic_ai.models.instrumented import InstrumentedModel
-
-    pydantic_ai_installed = True
-except:
-    pydantic_ai_installed = False
-
-
-def is_pydantic_ai_installed():
-    if not pydantic_ai_installed:
-        raise ImportError(
-            "Pydantic AI is not installed. Please install it with `pip install pydantic-ai`."
-        )
-
-
-class PydanticAIAgent(Agent):
-    def __init__(self, *args, **kwargs):
-        with capture_tracing_integration("pydantic_ai.agent.PydanticAIAgent"):
-            is_pydantic_ai_installed()
-            is_opentelemetry_available()
-
-            super().__init__(*args, **kwargs)
-
-            # attributes to be set if ran synchronously
-            self.metric_collection: str = None
-            self.metrics: list[BaseMetric] = None
-
-            # trace attributes to be set if ran synchronously
-            self._trace_name: str = None
-            self._trace_tags: list[str] = None
-            self._trace_metadata: dict = None
-            self._trace_thread_id: str = None
-            self._trace_user_id: str = None
-
-            # Patch the run method only for this instance
-            self._patch_run_method()
-            self._patch_run_method_sync()
-            self._patch_tool_decorator()
-
-    def _patch_tool_decorator(self):
-        """Patch the tool decorator to print input and output"""
-        original_tool = self.tool
-
-        @functools.wraps(original_tool)
-        def patched_tool(
-            *args,
-            metric_collection: Optional[str] = None,
-            metrics: Optional[List[BaseMetric]] = None,
-            **kwargs
-        ):
-
-            # Check if function is in args (direct decoration: @agent.tool)
-            if args and callable(args[0]):
-                original_func = args[0]
-                patched_func = self._create_patched_function(
-                    original_func, metric_collection, metrics
-                )
-                new_args = (patched_func,) + args[1:]
-                result = original_tool(*new_args, **kwargs)
-                return result
-            else:
-                # Decorator called with parameters: @agent.tool(metric_collection="...")
-                # Return a decorator that will receive the function
-                def decorator_with_params(func):
-                    patched_func = self._create_patched_function(
-                        func, metric_collection, metrics
-                    )
-                    return original_tool(patched_func, **kwargs)
-
-                return decorator_with_params
-
-        # Replace the tool method for this instance
-        self.tool = patched_tool
-
-    def _create_patched_function(
-        self, original_func, metric_collection, metrics
-    ):
-        """Create a patched version of the function that adds tracing"""
-        if inspect.iscoroutinefunction(original_func):
-
-            @functools.wraps(original_func)
-            async def patched_async_func(*func_args, **func_kwargs):
-                result = await original_func(*func_args, **func_kwargs)
-
-                current_span = trace.get_current_span()
-                if current_span.is_recording():
-                    try:
-                        result_str = str(result)
-                    except Exception:
-                        result_str = ""
-                    current_span.set_attribute(
-                        "confident.span.output", result_str
-                    )
-                    if metric_collection:
-                        current_span.set_attribute(
-                            "confident.span.metric_collection",
-                            metric_collection,
-                        )
-                    # TODO: add metrics in component level evals
-                return result
-
-            return patched_async_func
-        else:
-
-            @functools.wraps(original_func)
-            def patched_sync_func(*func_args, **func_kwargs):
-                result = original_func(*func_args, **func_kwargs)
-
-                current_span = trace.get_current_span()
-                if current_span.is_recording():
-                    try:
-                        result_str = str(result)
-                    except Exception:
-                        result_str = ""
-                    current_span.set_attribute(
-                        "confident.span.output", result_str
-                    )
-                    if metric_collection:
-                        current_span.set_attribute(
-                            "confident.span.metric_collection",
-                            metric_collection,
-                        )
-                    # TODO: add metrics in component level evals
-                return result
-
-            return patched_sync_func
-
-    def _patch_run_method(self):
-        """Patch the Agent.run method only for this PydanticAIAgent instance"""
-        original_run = self.run
-
-        @functools.wraps(original_run)
-        async def patched_run(
-            *args,
-            metric_collection=None,
-            metrics=None,
-            trace_name=None,
-            trace_tags=None,
-            trace_metadata=None,
-            trace_thread_id=None,
-            trace_user_id=None,
-            **kwargs
-        ):
-            # extract and validate flattened arguments - use safe pop with defaults
-            metric_collection = parse_string(metric_collection)
-            trace_name = parse_string(trace_name)
-            trace_tags = parse_list_of_strings(trace_tags)
-            trace_thread_id = parse_string(trace_thread_id)
-            trace_user_id = parse_string(trace_user_id)
-
-            if metrics is not None and not (
-                isinstance(metrics, list)
-                and all(isinstance(m, BaseMetric) for m in metrics)
-            ):
-                raise TypeError(
-                    "metrics must be a list of BaseMetric instances"
-                )
-
-            if trace_metadata is not None and not isinstance(
-                trace_metadata, dict
-            ):
-                raise TypeError("trace_metadata must be a dictionary")
-
-            model = kwargs.get("model", None)
-            infer_name = kwargs.get("infer_name", True)
-
-            if infer_name and self.name is None:
-                self._infer_name(inspect.currentframe())
-            model_used = self._get_model(model)
-            del model
-
-            if isinstance(model_used, InstrumentedModel):
-                tracer = model_used.instrumentation_settings.tracer
-            else:
-                tracer = NoOpTracer()
-
-            with tracer.start_as_current_span("agent") as run_span:
-                result = await original_run(*args, **kwargs)
-
-                name = "agent"
-                if self.name:
-                    name = str(self.name)
-
-                input = ""
-                if isinstance(args[0], str):
-                    input = args[0]
-                elif isinstance(args[0], list) and all(
-                    isinstance(i, str) for i in args[0]
-                ):
-                    input = args[0]
-
-                output = ""
-                try:
-                    output = str(result.output)
-                except Exception:
-                    pass
-
-                # set agent span attributes
-                run_span.set_attribute("confident.span.type", "agent")
-                run_span.set_attribute("confident.agent.name", name)
-                run_span.set_attribute("confident.agent.input", input)
-                run_span.set_attribute("confident.agent.output", output)
-
-                # fallback for input and output not being set
-                run_span.set_attribute("confident.span.input", input)
-                run_span.set_attribute("confident.span.output", output)
-
-                if metric_collection:  # flattened argument to be replaced
-                    run_span.set_attribute(
-                        "confident.span.metric_collection", metric_collection
-                    )
-                elif self.metric_collection:  # for run_sync
-                    run_span.set_attribute(
-                        "confident.span.metric_collection",
-                        self.metric_collection,
-                    )
-
-                # set the flattened trace attributes
-                if trace_name:
-                    run_span.set_attribute("confident.trace.name", trace_name)
-                if trace_tags:
-                    run_span.set_attribute("confident.trace.tags", trace_tags)
-                if trace_metadata:
-                    run_span.set_attribute(
-                        "confident.trace.metadata", json.dumps(trace_metadata)
-                    )
-                if trace_thread_id:
-                    run_span.set_attribute(
-                        "confident.trace.thread_id", trace_thread_id
-                    )
-                if trace_user_id:
-                    run_span.set_attribute(
-                        "confident.trace.user_id", trace_user_id
-                    )
-
-                # for run_sync
-                if self._trace_name:
-                    run_span.set_attribute(
-                        "confident.trace.name", self._trace_name
-                    )
-                if self._trace_tags:
-                    run_span.set_attribute(
-                        "confident.trace.tags", self._trace_tags
-                    )
-                if self._trace_metadata:
-                    run_span.set_attribute(
-                        "confident.trace.metadata",
-                        json.dumps(self._trace_metadata),
-                    )
-                if self._trace_thread_id:
-                    run_span.set_attribute(
-                        "confident.trace.thread_id", self._trace_thread_id
-                    )
-                if self._trace_user_id:
-                    run_span.set_attribute(
-                        "confident.trace.user_id", self._trace_user_id
-                    )
-
-                if metrics:  # flattened argument to be replaced
-                    trace_manager.test_case_metrics.append(
-                        TestCaseMetricPair(
-                            test_case=LLMTestCase(
-                                input=input, actual_output=output
-                            ),
-                            metrics=metrics,
-                        )
-                    )
-                elif self.metrics:  # for run_sync
-                    trace_manager.test_case_metrics.append(
-                        TestCaseMetricPair(
-                            test_case=LLMTestCase(
-                                input=input, actual_output=output
-                            ),
-                            metrics=self.metrics,
-                        )
-                    )
-
-            return result
-
-        # Replace the method only for this instance
-        self.run = patched_run
-
-    def _patch_run_method_sync(self):
-        """Patch the Agent.run method only for this PydanticAIAgent instance"""
-        original_run = self.run_sync
-
-        @functools.wraps(original_run)
-        def patched_run(
-            *args,
-            metric_collection=None,
-            metrics=None,
-            trace_name=None,
-            trace_tags=None,
-            trace_metadata=None,
-            trace_thread_id=None,
-            trace_user_id=None,
-            **kwargs
-        ):
-            metric_collection = parse_string(metric_collection)
-            trace_name = parse_string(trace_name)
-            trace_tags = parse_list_of_strings(trace_tags)
-            trace_thread_id = parse_string(trace_thread_id)
-            trace_user_id = parse_string(trace_user_id)
-
-            if metrics is not None and not (
-                isinstance(metrics, list)
-                and all(isinstance(m, BaseMetric) for m in metrics)
-            ):
-                raise TypeError(
-                    "metrics must be a list of BaseMetric instances"
-                )
-
-            if trace_metadata is not None and not isinstance(
-                trace_metadata, dict
-            ):
-                raise TypeError("trace_metadata must be a dictionary")
-
-            # attributes to be set if ran synchronously
-            if metric_collection:
-                self.metric_collection = metric_collection
-            if metrics:
-                self.metrics = metrics
-
-            self._trace_name = trace_name
-            self._trace_tags = trace_tags
-            self._trace_metadata = trace_metadata
-            self._trace_thread_id = trace_thread_id
-            self._trace_user_id = trace_user_id
-
-            result = original_run(*args, **kwargs)
-
-            return result
-
-        # Replace the method only for this instance
-        self.run_sync = patched_run