deepeval 3.4.9__tar.gz → 3.5.1__tar.gz

{deepeval-3.4.9 → deepeval-3.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.4.9
+Version: 3.5.1
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -189,16 +189,6 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 ```
 pip install -U deepeval
 ```
-### Environment variables (.env / .env.local)
-
-DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
-**Precedence:** process env -> `.env.local` -> `.env`.
-Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
-
-```bash
-cp .env.example .env.local
-# then edit .env.local (ignored by git)
-```
 
 ## Create an account (highly recommended)
 
@@ -391,9 +381,20 @@ evaluate(dataset, [answer_relevancy_metric])
 dataset.evaluate([answer_relevancy_metric])
 ```
 
-# LLM Evaluation With Confident AI
+## A Note on Env Variables (.env / .env.local)
+
+DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
+**Precedence:** process env -> `.env.local` -> `.env`.
+Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+```
+
+# DeepEval With Confident AI
 
-The correct LLM evaluation lifecycle is only achievable with [the DeepEval platform](https://confident-ai.com?utm_source=Github). It allows you to:
+DeepEval's cloud platform, [Confident AI](https://confident-ai.com?utm_source=Github), allows you to:
 
 1. Curate/annotate evaluation datasets on the cloud
 2. Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best

{deepeval-3.4.9 → deepeval-3.5.1}/README.md

@@ -140,16 +140,6 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 ```
 pip install -U deepeval
 ```
-### Environment variables (.env / .env.local)
-
-DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
-**Precedence:** process env -> `.env.local` -> `.env`.
-Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
-
-```bash
-cp .env.example .env.local
-# then edit .env.local (ignored by git)
-```
 
 ## Create an account (highly recommended)
 
@@ -342,9 +332,20 @@ evaluate(dataset, [answer_relevancy_metric])
 dataset.evaluate([answer_relevancy_metric])
 ```
 
-# LLM Evaluation With Confident AI
+## A Note on Env Variables (.env / .env.local)
+
+DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
+**Precedence:** process env -> `.env.local` -> `.env`.
+Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+```
+
+# DeepEval With Confident AI
 
-The correct LLM evaluation lifecycle is only achievable with [the DeepEval platform](https://confident-ai.com?utm_source=Github). It allows you to:
+DeepEval's cloud platform, [Confident AI](https://confident-ai.com?utm_source=Github), allows you to:
 
 1. Curate/annotate evaluation datasets on the cloud
 2. Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best

deepeval-3.5.1/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.5.1"

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/benchmarks/drop/drop.py

@@ -1,6 +1,5 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
-from typing import Union
 
 from deepeval.dataset import Golden
 from deepeval.benchmarks.base_benchmark import (
@@ -50,7 +49,7 @@ class DROP(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/benchmarks/hellaswag/hellaswag.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -51,7 +51,7 @@ class HellaSwag(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/benchmarks/logi_qa/logi_qa.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 import requests
 import json
@@ -52,7 +52,7 @@ class LogiQA(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/benchmarks/math_qa/math_qa.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -50,7 +50,7 @@ class MathQA(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/benchmarks/mmlu/mmlu.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -49,7 +49,7 @@ class MMLU(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/benchmarks/truthful_qa/truthful_qa.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -59,7 +59,7 @@ class TruthfulQA(DeepEvalBaseBenchmark):
         self,
         model: DeepEvalBaseLLM,
         *args,
-        batch_size: int | None = None,
+        batch_size: Union[int, None] = None,
         **kwargs,
     ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/confident/api.py

@@ -10,6 +10,7 @@ from tenacity import (
     retry_if_exception_type,
     RetryCallState,
 )
+from pydantic import SecretStr
 
 import deepeval
 from deepeval.key_handler import KEY_FILE_HANDLER, KeyValues
@@ -88,7 +89,9 @@ class Endpoints(Enum):
     TEST_RUN_ENDPOINT = "/v1/test-run"
     TRACES_ENDPOINT = "/v1/traces"
     ANNOTATIONS_ENDPOINT = "/v1/annotations"
+    PROMPTS_VERSION_ID_ENDPOINT = "/v1/prompts/:alias/versions/:versionId"
     PROMPTS_ENDPOINT = "/v1/prompts"
+    PROMPTS_VERSIONS_ENDPOINT = "/v1/prompts/:alias/versions"
     SIMULATE_ENDPOINT = "/v1/simulate"
     EVALUATE_ENDPOINT = "/v1/evaluate"
 

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/integrations/langchain/callback.py

@@ -9,6 +9,7 @@ from deepeval.tracing.types import (
 from deepeval.metrics import BaseMetric, TaskCompletionMetric
 from deepeval.test_case import LLMTestCase
 from deepeval.test_run import global_test_run_manager
+import uuid
 
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
@@ -81,6 +82,26 @@ class CallbackHandler(BaseCallbackHandler):
             )
             super().__init__()
 
+    def on_llm_new_token(
+        self,
+        token: str,
+        *,
+        chunk,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        **kwargs: Any,
+    ):
+        llm_span: Optional[LlmSpan] = trace_manager.get_span_by_uuid(
+            str(run_id)
+        )
+        if llm_span is None:
+            return
+        if llm_span.token_intervals is None:
+            llm_span.token_intervals = {perf_counter(): token}
+        else:
+            llm_span.token_intervals[perf_counter()] = token
+
     def check_active_trace_id(self):
         if self.active_trace_id is None:
             self.active_trace_id = trace_manager.start_new_trace().uuid

deepeval-3.5.1/deepeval/integrations/pydantic_ai/__init__.py

@@ -0,0 +1,3 @@
+from .patcher import instrument as instrument_pydantic_ai
+
+__all__ = ["instrument_pydantic_ai"]

deepeval-3.4.9/deepeval/integrations/pydantic_ai/setup.py → deepeval-3.5.1/deepeval/integrations/pydantic_ai/otel.py

@@ -31,14 +31,6 @@ def instrument_pydantic_ai(api_key: Optional[str] = None):
     with capture_tracing_integration("pydantic_ai"):
         is_opentelemetry_available()
 
-        if api_key:
-            deepeval.login(api_key)
-
-        api_key = get_confident_api_key()
-
-        if not api_key:
-            raise ValueError("No api key provided.")
-
         # create a new tracer provider
         tracer_provider = TracerProvider()
         tracer_provider.add_span_processor(

deepeval-3.5.1/deepeval/integrations/pydantic_ai/patcher.py

@@ -0,0 +1,376 @@
+import functools
+import deepeval
+from deepeval.tracing.types import LlmOutput, LlmToolCall
+from pydantic_ai.agent import AgentRunResult
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import AgentSpan, LlmSpan
+from deepeval.tracing.tracing import Observer
+from typing import List, Callable, Optional, Any
+from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.metrics.base_metric import BaseMetric
+from deepeval.confident.api import get_confident_api_key
+from deepeval.integrations.pydantic_ai.otel import instrument_pydantic_ai
+from deepeval.telemetry import capture_tracing_integration
+from deepeval.prompt import Prompt
+
+try:
+    from pydantic_ai.agent import Agent
+    from pydantic_ai.models import Model
+    from pydantic_ai.messages import (
+        ModelResponse,
+        ModelRequest,
+        ModelResponsePart,
+        TextPart,
+        ToolCallPart,
+        SystemPromptPart,
+        ToolReturnPart,
+        UserPromptPart,
+    )
+
+    pydantic_ai_installed = True
+except:
+    pydantic_ai_installed = True
+
+
+def _patch_agent_tool_decorator():
+    original_tool = Agent.tool
+
+    @functools.wraps(original_tool)
+    def wrapper(
+        *args,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        **kwargs
+    ):
+        # Case 1: Direct decoration - @agent.tool
+        if args and callable(args[0]):
+            patched_func = _create_patched_tool(
+                args[0], metrics, metric_collection
+            )
+            new_args = (patched_func,) + args[1:]
+            return original_tool(*new_args, **kwargs)
+
+        # Case 2: Decoration with arguments - @agent.tool(metrics=..., metric_collection=...)
+        else:
+            # Return a decorator function that will receive the actual function
+            def decorator(func):
+                patched_func = _create_patched_tool(
+                    func, metrics, metric_collection
+                )
+                return original_tool(*args, **kwargs)(patched_func)
+
+            return decorator
+
+    Agent.tool = wrapper
+
+
+def _create_patched_tool(
+    func: Callable,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
+    import asyncio
+
+    original_func = func
+
+    is_async = asyncio.iscoroutinefunction(original_func)
+
+    if is_async:
+
+        @functools.wraps(original_func)
+        async def async_wrapper(*args, **kwargs):
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": args, **kwargs},
+            ) as observer:
+                result = await original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return async_wrapper
+    else:
+
+        @functools.wraps(original_func)
+        def sync_wrapper(*args, **kwargs):
+            with Observer(
+                span_type="tool",
+                func_name=original_func.__name__,
+                metrics=metrics,
+                metric_collection=metric_collection,
+                function_kwargs={"args": args, **kwargs},
+            ) as observer:
+                result = original_func(*args, **kwargs)
+                observer.result = result
+
+            return result
+
+        return sync_wrapper
+
+
+def _patch_agent_init():
+    original_init = Agent.__init__
+
+    @functools.wraps(original_init)
+    def wrapper(
+        self,
+        *args,
+        llm_metric_collection: Optional[str] = None,
+        llm_metrics: Optional[List[BaseMetric]] = None,
+        llm_prompt: Optional[Prompt] = None,
+        agent_metric_collection: Optional[str] = None,
+        agent_metrics: Optional[List[BaseMetric]] = None,
+        **kwargs
+    ):
+        result = original_init(self, *args, **kwargs)
+        _patch_llm_model(
+            self._model, llm_metric_collection, llm_metrics, llm_prompt
+        )  # runtime patch of the model
+        _patch_agent_run(agent_metric_collection, agent_metrics)
+        return result
+
+    Agent.__init__ = wrapper
+
+
+def _patch_agent_run(
+    agent_metric_collection: Optional[str] = None,
+    agent_metrics: Optional[List[BaseMetric]] = None,
+):
+    original_run = Agent.run
+
+    @functools.wraps(original_run)
+    async def wrapper(
+        *args,
+        trace_metric_collection: Optional[str] = None,
+        trace_metrics: Optional[List[BaseMetric]] = None,
+        trace_name: Optional[str] = None,
+        trace_tags: Optional[List[str]] = None,
+        trace_metadata: Optional[dict] = None,
+        trace_thread_id: Optional[str] = None,
+        trace_user_id: Optional[str] = None,
+        **kwargs
+    ):
+        with Observer(
+            span_type="agent",
+            func_name="Agent",
+            function_kwargs={"input": args[1]},
+            metrics=agent_metrics,
+            metric_collection=agent_metric_collection,
+        ) as observer:
+            result = await original_run(*args, **kwargs)
+            observer.update_span_properties = (
+                lambda agent_span: set_agent_span_attributes(agent_span, result)
+            )
+            observer.result = result.output
+
+            _update_trace_context(
+                trace_name=trace_name,
+                trace_tags=trace_tags,
+                trace_metadata=trace_metadata,
+                trace_thread_id=trace_thread_id,
+                trace_user_id=trace_user_id,
+                trace_metric_collection=trace_metric_collection,
+                trace_metrics=trace_metrics,
+                trace_input=args[1],
+                trace_output=result.output,
+            )
+
+        return result
+
+    Agent.run = wrapper
+
+
+def _update_trace_context(
+    trace_name: Optional[str] = None,
+    trace_tags: Optional[List[str]] = None,
+    trace_metadata: Optional[dict] = None,
+    trace_thread_id: Optional[str] = None,
+    trace_user_id: Optional[str] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+    trace_input: Optional[Any] = None,
+    trace_output: Optional[Any] = None,
+):
+
+    current_trace = current_trace_context.get()
+    current_trace.name = trace_name
+    current_trace.tags = trace_tags
+    current_trace.metadata = trace_metadata
+    current_trace.thread_id = trace_thread_id
+    current_trace.user_id = trace_user_id
+    current_trace.metric_collection = trace_metric_collection
+    current_trace.metrics = trace_metrics
+    current_trace.input = trace_input
+    current_trace.output = trace_output
+
+
+def _patch_llm_model(
+    model: Model,
+    llm_metric_collection: Optional[str] = None,
+    llm_metrics: Optional[List[BaseMetric]] = None,
+    llm_prompt: Optional[Prompt] = None,
+):
+    original_func = model.request
+    try:
+        model_name = model.model_name
+    except Exception:
+        model_name = "unknown"
+
+    @functools.wraps(original_func)
+    async def wrapper(*args, **kwargs):
+        with Observer(
+            span_type="llm",
+            func_name="LLM",
+            observe_kwargs={"model": model_name},
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+        ) as observer:
+            result = await original_func(*args, **kwargs)
+            request = kwargs.get("messages", [])
+            if not request:
+                request = args[0]
+            observer.update_span_properties = (
+                lambda llm_span: set_llm_span_attributes(
+                    llm_span, args[0], result, llm_prompt
+                )
+            )
+            observer.result = result
+        return result
+
+    model.request = wrapper
+
+
+def instrument(otel: Optional[bool] = False, api_key: Optional[str] = None):
+
+    if api_key:
+        deepeval.login(api_key)
+
+    api_key = get_confident_api_key()
+
+    if not api_key:
+        raise ValueError("No api key provided.")
+
+    if otel:
+        instrument_pydantic_ai(api_key)
+    else:
+        with capture_tracing_integration("pydantic_ai"):
+            _patch_agent_init()
+            _patch_agent_tool_decorator()
+
+
+def set_llm_span_attributes(
+    llm_span: LlmSpan,
+    requests: List[ModelRequest],
+    result: ModelResponse,
+    llm_prompt: Optional[Prompt] = None,
+):
+    llm_span.prompt = llm_prompt
+
+    input = []
+    for request in requests:
+        for part in request.parts:
+            if isinstance(part, SystemPromptPart):
+                input.append({"role": "System", "content": part.content})
+            elif isinstance(part, UserPromptPart):
+                input.append({"role": "User", "content": part.content})
+            elif isinstance(part, ToolCallPart):
+                input.append(
+                    {
+                        "role": "Tool Call",
+                        "name": part.tool_name,
+                        "content": part.args_as_json_str(),
+                    }
+                )
+            elif isinstance(part, ToolReturnPart):
+                input.append(
+                    {
+                        "role": "Tool Return",
+                        "name": part.tool_name,
+                        "content": part.model_response_str(),
+                    }
+                )
+    llm_span.input = input
+
+    content = ""
+    tool_calls = []
+    for part in result.parts:
+        if isinstance(part, TextPart):
+            content += part.content + "\n"
+        elif isinstance(part, ToolCallPart):
+            tool_calls.append(
+                LlmToolCall(name=part.tool_name, args=part.args_as_dict())
+            )
+    llm_span.output = LlmOutput(
+        role="Assistant", content=content, tool_calls=tool_calls
+    )
+    llm_span.tools_called = _extract_tools_called_from_llm_response(
+        result.parts
+    )
+
+
+def set_agent_span_attributes(agent_span: AgentSpan, result: AgentRunResult):
+    agent_span.tools_called = _extract_tools_called(result)
+
+
+# llm tools called
+def _extract_tools_called_from_llm_response(
+    result: List[ModelResponsePart],
+) -> List[ToolCall]:
+    tool_calls = []
+
+    # Loop through each ModelResponsePart
+    for part in result:
+        # Look for parts with part_kind="tool-call"
+        if hasattr(part, "part_kind") and part.part_kind == "tool-call":
+            # Extract tool name and args from the ToolCallPart
+            tool_name = part.tool_name
+            input_parameters = (
+                part.args_as_dict() if hasattr(part, "args_as_dict") else None
+            )
+
+            # Create and append ToolCall object
+            tool_call = ToolCall(
+                name=tool_name, input_parameters=input_parameters
+            )
+            tool_calls.append(tool_call)
+
+    return tool_calls
+
+
+# TODO: llm tools called (reposne is present next message)
+def _extract_tools_called(result: AgentRunResult) -> List[ToolCall]:
+    tool_calls = []
+
+    # Access the message history from the _state
+    message_history = result._state.message_history
+
+    # Scan through all messages in the history
+    for message in message_history:
+        # Check if this is a ModelResponse (kind="response")
+        if hasattr(message, "kind") and message.kind == "response":
+            # For ModelResponse messages, check each part
+            if hasattr(message, "parts"):
+                for part in message.parts:
+                    # Look for parts with part_kind="tool-call"
+                    if (
+                        hasattr(part, "part_kind")
+                        and part.part_kind == "tool-call"
+                    ):
+                        # Extract tool name and args from the ToolCallPart
+                        tool_name = part.tool_name
+                        input_parameters = (
+                            part.args_as_dict()
+                            if hasattr(part, "args_as_dict")
+                            else None
+                        )
+
+                        # Create and append ToolCall object
+                        tool_call = ToolCall(
+                            name=tool_name, input_parameters=input_parameters
+                        )
+                        tool_calls.append(tool_call)
+
+    return tool_calls

{deepeval-3.4.9 → deepeval-3.5.1}/deepeval/metrics/__init__.py

@@ -69,7 +69,7 @@ __all__ = [
     "ConversationalGEval",
     "DAGMetric",
     "DeepAcyclicGraph",
-    "ConversationalDAGMetric"
+    "ConversationalDAGMetric",
     # RAG metrics
     "AnswerRelevancyMetric",
     "FaithfulnessMetric",