deepeval 3.4.6__tar.gz → 3.4.7__tar.gz

{deepeval-3.4.6 → deepeval-3.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.4.6
+Version: 3.4.7
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -31,6 +31,7 @@ Requires-Dist: pytest-asyncio
 Requires-Dist: pytest-repeat
 Requires-Dist: pytest-rerunfailures (>=12.0,<13.0)
 Requires-Dist: pytest-xdist
+Requires-Dist: python-dotenv (>=1.1.1,<2.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: rich (>=13.6.0,<15.0.0)
 Requires-Dist: sentry-sdk
@@ -186,6 +187,16 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 ```
 pip install -U deepeval
 ```
+### Environment variables (.env / .env.local)
+
+DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
+**Precedence:** process env -> `.env.local` -> `.env`.
+Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+```
 
 ## Create an account (highly recommended)
 
@@ -411,6 +422,20 @@ You should see a link displayed in the CLI once the test has finished running. P
 
 <br />
 
+## Configuration
+
+### Environment variables via .env files
+
+Using `.env.local` or `.env` is optional. If they are missing, DeepEval uses your existing environment variables. When present, dotenv environment variables are auto-loaded at import time (unless you set `DEEPEVAL_DISABLE_DOTENV=1`).
+
+**Precedence:** process env -> `.env.local` -> `.env`
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+
+<br />
+
 # Contributing
 
 Please read [CONTRIBUTING.md](https://github.com/confident-ai/deepeval/blob/main/CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.

{deepeval-3.4.6 → deepeval-3.4.7}/README.md

@@ -140,6 +140,16 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 ```
 pip install -U deepeval
 ```
+### Environment variables (.env / .env.local)
+
+DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
+**Precedence:** process env -> `.env.local` -> `.env`.
+Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+```
 
 ## Create an account (highly recommended)
 
@@ -365,6 +375,20 @@ You should see a link displayed in the CLI once the test has finished running. P
 
 <br />
 
+## Configuration
+
+### Environment variables via .env files
+
+Using `.env.local` or `.env` is optional. If they are missing, DeepEval uses your existing environment variables. When present, dotenv environment variables are auto-loaded at import time (unless you set `DEEPEVAL_DISABLE_DOTENV=1`).
+
+**Precedence:** process env -> `.env.local` -> `.env`
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+
+<br />
+
 # Contributing
 
 Please read [CONTRIBUTING.md](https://github.com/confident-ai/deepeval/blob/main/CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/__init__.py

@@ -2,6 +2,11 @@ import os
 import warnings
 import re
 
+# load environment variables before other imports
+from .env import autoload_dotenv as _autoload_dotenv
+
+_autoload_dotenv()
+
 # Optionally add telemetry
 from ._version import __version__
 
@@ -11,7 +16,7 @@ from deepeval.test_run import on_test_run_end, log_hyperparameters
 from deepeval.utils import login
 from deepeval.telemetry import *
 
-if os.getenv("DEEPEVAL_GRPC_LOGGING") != "YES":
+if os.getenv("DEEPEVAL_GRPC_LOGGING") != "1":
     os.environ["GRPC_VERBOSITY"] = "ERROR"
     os.environ["GRPC_TRACE"] = ""
 
@@ -61,7 +66,7 @@ def check_for_update():
 
 
 def update_warning_opt_in():
-    return os.getenv("DEEPEVAL_UPDATE_WARNING_OPT_IN") == "YES"
+    return os.getenv("DEEPEVAL_UPDATE_WARNING_OPT_IN") == "1"
 
 
 def is_read_only_env():

deepeval-3.4.7/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.4.7"

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/cli/main.py

@@ -154,7 +154,7 @@ def view():
 
 @app.command(name="enable-grpc-logging")
 def enable_grpc_logging():
-    os.environ["DEEPEVAL_GRPC_LOGGING"] = "YES"
+    os.environ["DEEPEVAL_GRPC_LOGGING"] = "1"
 
 
 #############################################

deepeval-3.4.7/deepeval/env.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+import os
+
+try:
+    from dotenv import load_dotenv, find_dotenv  # type: ignore
+except Exception:
+    load_dotenv = None
+    find_dotenv = None
+
+
+def autoload_dotenv() -> None:
+    """
+    Autoload environment variables for DeepEval at import time.
+
+    Precedence from highest -> lowest:
+      1) Existing process environment variables
+      2) .env.local (from current working directory)
+      3) .env (from current working directory)
+
+    Behavior:
+      - Loads .env.local then .env if present, without overriding existing vars.
+      - Opt-out by setting DEEPEVAL_DISABLE_DOTENV=1.
+      - Soft-fails cleanly if python-dotenv is not installed.
+    """
+    if os.getenv("DEEPEVAL_DISABLE_DOTENV") == "1":
+        return
+
+    if not (load_dotenv and find_dotenv):
+        return
+
+    for name in (".env.local", ".env"):
+        path = find_dotenv(name, usecwd=True)
+        if path:
+            # Don't override previously set values
+            load_dotenv(path, override=False)

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/evaluate/utils.py

@@ -24,8 +24,9 @@ from deepeval.test_run import (
     MetricData,
 )
 from deepeval.evaluate.types import TestResult
-from deepeval.tracing.api import TraceApi, BaseApiSpan
+from deepeval.tracing.api import TraceApi, BaseApiSpan, TraceSpanApiStatus
 from deepeval.tracing.tracing import BaseSpan, Trace
+from deepeval.tracing.types import TraceSpanStatus
 from deepeval.constants import PYTEST_RUN_TEST_NAME
 from deepeval.tracing.utils import (
     perf_counter_to_datetime,
@@ -247,6 +248,11 @@ def create_api_trace(trace: Trace, golden: Golden) -> TraceApi:
         tools_called=trace.tools_called,
         expected_tools=trace.expected_tools,
         metadata=golden.additional_metadata,
+        status=(
+            TraceSpanApiStatus.SUCCESS
+            if trace.status == TraceSpanStatus.SUCCESS
+            else TraceSpanApiStatus.ERRORED
+        ),
     )
 
 

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/integrations/langchain/__init__.py

@@ -1,4 +1,5 @@
 from .callback import CallbackHandler
+from .patch import tool
 
 
 __all__ = ["CallbackHandler"]

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/integrations/langchain/callback.py

@@ -248,6 +248,8 @@ class CallbackHandler(BaseCallbackHandler):
             model=model,
             # fallback for on_end callback
             end_time=perf_counter(),
+            metric_collection=metadata.get("metric_collection", None),
+            metrics=metadata.get("metrics", None),
         )
 
         self.add_span_to_trace(llm_span)
@@ -348,6 +350,8 @@ class CallbackHandler(BaseCallbackHandler):
             ),
             # fallback for on_end callback
             end_time=perf_counter(),
+            metric_collection=metadata.get("metric_collection", None),
+            metrics=metadata.get("metrics", None),
         )
         self.add_span_to_trace(tool_span)
 

deepeval-3.4.7/deepeval/integrations/langchain/patch.py

@@ -0,0 +1,32 @@
+from langchain_core.tools import tool as original_tool, BaseTool
+from deepeval.metrics import BaseMetric
+from typing import List, Optional, Callable, Any
+from functools import wraps
+
+
+def tool(
+    *args,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+    **kwargs
+):
+    """
+    Patched version of langchain_core.tools.tool that prints inputs and outputs
+    """
+
+    # original_tool returns a decorator function, so we need to return a decorator
+    def decorator(func: Callable) -> BaseTool:
+
+        # Apply the original tool decorator to get the BaseTool
+        tool_instance = original_tool(*args, **kwargs)(func)
+
+        if isinstance(tool_instance, BaseTool):
+            if tool_instance.metadata is None:
+                tool_instance.metadata = {}
+
+            tool_instance.metadata["metric_collection"] = metric_collection
+            tool_instance.metadata["metrics"] = metrics
+
+        return tool_instance
+
+    return decorator

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/integrations/pydantic_ai/agent.py

@@ -1,6 +1,6 @@
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.metrics import BaseMetric
-from typing import List
+from typing import List, Optional
 import functools
 import inspect
 import json
@@ -8,6 +8,7 @@ from deepeval.test_case import LLMTestCase
 from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.tracing.tracing import trace_manager
 from deepeval.tracing.otel.utils import parse_string, parse_list_of_strings
+from opentelemetry import trace
 
 try:
     from opentelemetry.trace import NoOpTracer
@@ -63,6 +64,95 @@ class PydanticAIAgent(Agent):
             # Patch the run method only for this instance
             self._patch_run_method()
             self._patch_run_method_sync()
+            self._patch_tool_decorator()
+
+    def _patch_tool_decorator(self):
+        """Patch the tool decorator to print input and output"""
+        original_tool = self.tool
+
+        @functools.wraps(original_tool)
+        def patched_tool(
+            *args,
+            metric_collection: Optional[str] = None,
+            metrics: Optional[List[BaseMetric]] = None,
+            **kwargs
+        ):
+
+            # Check if function is in args (direct decoration: @agent.tool)
+            if args and callable(args[0]):
+                original_func = args[0]
+                patched_func = self._create_patched_function(
+                    original_func, metric_collection, metrics
+                )
+                new_args = (patched_func,) + args[1:]
+                result = original_tool(*new_args, **kwargs)
+                return result
+            else:
+                # Decorator called with parameters: @agent.tool(metric_collection="...")
+                # Return a decorator that will receive the function
+                def decorator_with_params(func):
+                    patched_func = self._create_patched_function(
+                        func, metric_collection, metrics
+                    )
+                    return original_tool(patched_func, **kwargs)
+
+                return decorator_with_params
+
+        # Replace the tool method for this instance
+        self.tool = patched_tool
+
+    def _create_patched_function(
+        self, original_func, metric_collection, metrics
+    ):
+        """Create a patched version of the function that adds tracing"""
+        if inspect.iscoroutinefunction(original_func):
+
+            @functools.wraps(original_func)
+            async def patched_async_func(*func_args, **func_kwargs):
+                result = await original_func(*func_args, **func_kwargs)
+
+                current_span = trace.get_current_span()
+                if current_span.is_recording():
+                    try:
+                        result_str = str(result)
+                    except Exception:
+                        result_str = ""
+                    current_span.set_attribute(
+                        "confident.span.output", result_str
+                    )
+                    if metric_collection:
+                        current_span.set_attribute(
+                            "confident.span.metric_collection",
+                            metric_collection,
+                        )
+                    # TODO: add metrics in component level evals
+                return result
+
+            return patched_async_func
+        else:
+
+            @functools.wraps(original_func)
+            def patched_sync_func(*func_args, **func_kwargs):
+                result = original_func(*func_args, **func_kwargs)
+
+                current_span = trace.get_current_span()
+                if current_span.is_recording():
+                    try:
+                        result_str = str(result)
+                    except Exception:
+                        result_str = ""
+                    current_span.set_attribute(
+                        "confident.span.output", result_str
+                    )
+                    if metric_collection:
+                        current_span.set_attribute(
+                            "confident.span.metric_collection",
+                            metric_collection,
+                        )
+                    # TODO: add metrics in component level evals
+                return result
+
+            return patched_sync_func
 
     def _patch_run_method(self):
         """Patch the Agent.run method only for this PydanticAIAgent instance"""

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/integrations/pydantic_ai/setup.py

@@ -49,7 +49,6 @@ def instrument_pydantic_ai(api_key: Optional[str] = None):
                 )
             )
         )
-        trace.set_tracer_provider(tracer_provider)
 
         # create an instrumented exporter
         from pydantic_ai.models.instrumented import InstrumentationSettings

deepeval-3.4.7/deepeval/openai_agents/__init__.py

@@ -0,0 +1,6 @@
+from deepeval.openai_agents.callback_handler import DeepEvalTracingProcessor
+from deepeval.openai_agents.runner import Runner
+from deepeval.openai_agents.patch import function_tool
+from deepeval.openai_agents.agent import DeepEvalAgent as Agent
+
+__all__ = ["DeepEvalTracingProcessor", "Runner", "function_tool", "Agent"]

deepeval-3.4.7/deepeval/openai_agents/agent.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import Any, Optional, Awaitable, Callable
+
+from deepeval.tracing import observe
+from deepeval.prompt import Prompt
+
+try:
+    from agents.agent import Agent as BaseAgent
+    from agents.models.interface import Model, ModelProvider
+except Exception as e:
+    raise RuntimeError(
+        "openai-agents is required for this integration. Please install it."
+    ) from e
+
+
+class _ObservedModel(Model):
+    def __init__(
+        self,
+        inner: Model,
+        *,
+        metrics: Optional[list[Any]] = None,
+        metric_collection: Optional[str] = None,
+        deepeval_prompt: Optional[Any] = None,
+    ) -> None:
+        self._inner = inner
+        self._metrics = metrics
+        self._metric_collection = metric_collection
+        self._deepeval_prompt = deepeval_prompt
+
+    # Delegate attributes not overridden
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._inner, name)
+
+    def _get_model_name(self) -> str:
+        try:
+            for attr in ("model", "model_name", "name"):
+                if hasattr(self._inner, attr):
+                    val = getattr(self._inner, attr)
+                    if val is not None:
+                        return str(val)
+        except Exception:
+            pass
+        return "unknown"
+
+    async def get_response(
+        self,
+        system_instructions,
+        input,
+        model_settings,
+        tools,
+        output_schema,
+        handoffs,
+        tracing,
+        *,
+        previous_response_id,
+        conversation_id,
+        prompt,
+    ):
+        model_name = self._get_model_name()
+
+        wrapped = observe(
+            metrics=self._metrics,
+            metric_collection=self._metric_collection,
+            type="llm",
+            model=model_name,
+            prompt=self._deepeval_prompt,
+        )(self._inner.get_response)
+
+        return await wrapped(
+            system_instructions,
+            input,
+            model_settings,
+            tools,
+            output_schema,
+            handoffs,
+            tracing,
+            previous_response_id=previous_response_id,
+            conversation_id=conversation_id,
+            prompt=prompt,
+        )
+
+    def stream_response(
+        self,
+        system_instructions,
+        input,
+        model_settings,
+        tools,
+        output_schema,
+        handoffs,
+        tracing,
+        *,
+        previous_response_id,
+        conversation_id,
+        prompt,
+    ):
+        # Optional: if you also want to observe streaming, uncomment and wrap similarly.
+        # wrapped = observe(
+        #     metrics=self._metrics,
+        #     metric_collection=self._metric_collection,
+        #     type="llm",
+        #     model=model_name,
+        # )(self._inner.stream_response)
+        # return wrapped(
+        #     system_instructions,
+        #     input,
+        #     model_settings,
+        #     tools,
+        #     output_schema,
+        #     handoffs,
+        #     tracing,
+        #     previous_response_id=previous_response_id,
+        #     conversation_id=conversation_id,
+        #     prompt=prompt,
+        # )
+        return self._inner.stream_response(
+            system_instructions,
+            input,
+            model_settings,
+            tools,
+            output_schema,
+            handoffs,
+            tracing,
+            previous_response_id=previous_response_id,
+            conversation_id=conversation_id,
+            prompt=prompt,
+        )
+
+
+class _ObservedProvider(ModelProvider):
+    def __init__(
+        self,
+        base: ModelProvider,
+        *,
+        metrics: Optional[list[Any]] = None,
+        metric_collection: Optional[str] = None,
+        deepeval_prompt: Optional[Any] = None,
+    ) -> None:
+        self._base = base
+        self._metrics = metrics
+        self._metric_collection = metric_collection
+        self._deepeval_prompt = deepeval_prompt
+
+    def get_model(self, model_name: str | None) -> Model:
+        model = self._base.get_model(model_name)
+        return _ObservedModel(
+            model,
+            metrics=self._metrics,
+            metric_collection=self._metric_collection,
+            deepeval_prompt=self._deepeval_prompt,
+        )
+
+
+@dataclass
+class DeepEvalAgent(BaseAgent[Any]):
+    """
+    A subclass of agents.Agent that accepts `metrics` and `metric_collection`
+    and ensures the underlying model's `get_response` is wrapped with deepeval.observe.
+    """
+
+    metrics: list[Any] | None = field(default=None)
+    metric_collection: str | None = field(default=None)
+    deepeval_prompt: Prompt | None = field(default=None)
+
+    def __post_init__(self):
+        super().__post_init__()
+        # If a direct Model instance is set on the agent, wrap it here.
+        if self.model is not None and not isinstance(self.model, str):
+            try:
+                from agents.models.interface import (
+                    Model as _Model,
+                )  # local import for safety
+
+                if isinstance(self.model, _Model):
+                    self.model = _ObservedModel(
+                        self.model,
+                        metrics=self.metrics,
+                        metric_collection=self.metric_collection,
+                        deepeval_prompt=self.deepeval_prompt,
+                    )
+            except Exception:
+                # If we can't import or wrap, silently skip.
+                pass

{deepeval-3.4.6 → deepeval-3.4.7}/deepeval/openai_agents/callback_handler.py

@@ -1,9 +1,9 @@
 from deepeval.tracing.tracing import (
     Observer,
-    SpanType,
-    current_trace_context,
+    current_span_context,
 )
 from deepeval.openai_agents.extractors import *
+from deepeval.tracing.context import current_trace_context
 
 try:
     from agents.tracing import Span, Trace, TracingProcessor
@@ -37,34 +37,41 @@ class DeepEvalTracingProcessor(TracingProcessor):
         self.span_observers: dict[str, Observer] = {}
 
     def on_trace_start(self, trace: "Trace") -> None:
-        observer = Observer(span_type=SpanType.AGENT, func_name=trace.name)
-        self.root_span_observers[trace.trace_id] = observer
-        observer.__enter__()
+        pass
 
     def on_trace_end(self, trace: "Trace") -> None:
-        # set thread id if exists
-        current_trace = current_trace_context.get()
-        thread_id = getattr(trace, "group_id", None)
-        current_trace.thread_id = thread_id
-
-        observer = self.root_span_observers.pop(trace.trace_id, None)
-        if observer:
-            observer.__exit__(None, None, None)
+        pass
 
     def on_span_start(self, span: "Span") -> None:
         if not span.started_at:
             return
         span_type = self.get_span_kind(span.span_data)
-        observer = Observer(span_type=span_type, func_name="NA")
-        if span_type == "llm":
-            observer.observe_kwargs["model"] = "temporary model"
-        observer.update_span_properties = (
-            lambda span_type: update_span_properties(span_type, span.span_data)
-        )
-        self.span_observers[span.span_id] = observer
-        observer.__enter__()
+        if span_type == "agent":
+            if isinstance(span.span_data, AgentSpanData):
+                current_trace = current_trace_context.get()
+                if current_trace:
+                    current_trace.name = span.span_data.name
+
+        if span_type == "tool":
+            return
+        elif span_type == "llm":
+            return
+        else:
+            observer = Observer(span_type=span_type, func_name="NA")
+            observer.update_span_properties = (
+                lambda base_span: update_span_properties(
+                    base_span, span.span_data
+                )
+            )
+            self.span_observers[span.span_id] = observer
+            observer.__enter__()
 
     def on_span_end(self, span: "Span") -> None:
+        span_type = self.get_span_kind(span.span_data)
+        if span_type == "llm":
+            current_span = current_span_context.get()
+            if current_span:
+                update_span_properties(current_span, span.span_data)
         observer = self.span_observers.pop(span.span_id, None)
         if observer:
             observer.__exit__(None, None, None)