deepeval 3.5.8__py3-none-any.whl → 3.6.0__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.5.8"
+__version__: str = "3.6.0"

deepeval/config/settings_manager.py

@@ -15,7 +15,7 @@ from enum import Enum
 from pydantic import SecretStr
 from deepeval.config.settings import get_settings, _SAVE_RE
 from deepeval.cli.dotenv_handler import DotenvHandler
-from deepeval.utils import bool_to_env_str
+from deepeval.config.utils import bool_to_env_str
 
 logger = logging.getLogger(__name__)
 StrOrEnum = Union[str, Enum]

deepeval/contextvars.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Optional
+
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden
+
+
+CURRENT_GOLDEN: ContextVar[Optional[Golden]] = ContextVar(
+    "CURRENT_GOLDEN", default=None
+)
+
+
+def set_current_golden(golden: Optional[Golden]):
+    return CURRENT_GOLDEN.set(golden)
+
+
+def get_current_golden() -> Optional[Golden]:
+    return CURRENT_GOLDEN.get()
+
+
+def reset_current_golden(token) -> None:
+    CURRENT_GOLDEN.reset(token)

deepeval/dataset/__init__.py

@@ -1,5 +1,11 @@
+from deepeval.contextvars import get_current_golden
 from .dataset import EvaluationDataset
 from .golden import Golden, ConversationalGolden
-from .test_run_tracer import init_global_test_run_tracer
 
-__all__ = ["EvaluationDataset", "Golden", "ConversationalGolden"]
+
+__all__ = [
+    "EvaluationDataset",
+    "Golden",
+    "ConversationalGolden",
+    "get_current_golden",
+]

deepeval/evaluate/execute.py

@@ -42,6 +42,7 @@ from deepeval.tracing.api import (
     BaseApiSpan,
 )
 from deepeval.dataset import Golden
+from deepeval.contextvars import set_current_golden, reset_current_golden
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.utils import copy_metrics
 from deepeval.utils import (
@@ -1480,6 +1481,7 @@ def execute_agentic_test_cases_from_loop(
         )
 
         for golden in goldens:
+            token = set_current_golden(golden)
             with capture_evaluation_run("golden"):
                 # yield golden
                 count += 1
@@ -1492,8 +1494,14 @@ def execute_agentic_test_cases_from_loop(
                     _progress=progress,
                     _pbar_callback_id=pbar_tags_id,
                 ):
-                    yield golden
-                    current_trace: Trace = current_trace_context.get()
+                    try:
+                        # yield golden to user code
+                        yield golden
+                        # control has returned from user code without error, capture trace now
+                        current_trace: Trace = current_trace_context.get()
+                    finally:
+                        # after user code returns control, always reset the context
+                        reset_current_golden(token)
 
                 update_pbar(progress, pbar_tags_id)
                 update_pbar(progress, pbar_id)
@@ -1849,6 +1857,7 @@ def a_execute_agentic_test_cases_from_loop(
 
         try:
             for index, golden in enumerate(goldens):
+                token = set_current_golden(golden)
                 current_golden_ctx.update(
                     {
                         "index": index,
@@ -1857,7 +1866,10 @@ def a_execute_agentic_test_cases_from_loop(
                     }
                 )
                 prev_task_length = len(created_tasks)
-                yield golden
+                try:
+                    yield golden
+                finally:
+                    reset_current_golden(token)
                 # if this golden created no tasks, bump bars now
                 if len(created_tasks) == prev_task_length:
                     update_pbar(progress, pbar_callback_id)

deepeval/integrations/pydantic_ai/__init__.py

@@ -1,5 +1,5 @@
 from .agent import DeepEvalPydanticAIAgent as Agent
-from .patcher import instrument as instrument_pydantic_ai
-from .otel import instrument_pydantic_ai as otel_instrument_pydantic_ai
+from .instrumentator import ConfidentInstrumentationSettings
+from .otel import instrument_pydantic_ai
 
-__all__ = ["instrument_pydantic_ai", "Agent", otel_instrument_pydantic_ai]
+__all__ = ["ConfidentInstrumentationSettings"]

deepeval/integrations/pydantic_ai/agent.py

@@ -1,339 +1,21 @@
-import inspect
-from typing import Optional, List, Generic, TypeVar
-from contextvars import ContextVar
-from contextlib import asynccontextmanager
-
-from deepeval.prompt import Prompt
-from deepeval.tracing.types import AgentSpan
-from deepeval.tracing.tracing import Observer
-from deepeval.metrics.base_metric import BaseMetric
-from deepeval.tracing.context import current_span_context
-from deepeval.integrations.pydantic_ai.utils import extract_tools_called
+import warnings
 
 try:
     from pydantic_ai.agent import Agent
-    from pydantic_ai.tools import AgentDepsT
-    from pydantic_ai.output import OutputDataT
-    from deepeval.integrations.pydantic_ai.utils import (
-        create_patched_tool,
-        update_trace_context,
-        patch_llm_model,
-    )
 
     is_pydantic_ai_installed = True
 except:
     is_pydantic_ai_installed = False
 
 
-def pydantic_ai_installed():
-    if not is_pydantic_ai_installed:
-        raise ImportError(
-            "Pydantic AI is not installed. Please install it with `pip install pydantic-ai`."
-        )
-
-
-_IS_RUN_SYNC = ContextVar("deepeval_is_run_sync", default=False)
-
-
-class DeepEvalPydanticAIAgent(
-    Agent[AgentDepsT, OutputDataT], Generic[AgentDepsT, OutputDataT]
-):
-
-    trace_name: Optional[str] = None
-    trace_tags: Optional[List[str]] = None
-    trace_metadata: Optional[dict] = None
-    trace_thread_id: Optional[str] = None
-    trace_user_id: Optional[str] = None
-    trace_metric_collection: Optional[str] = None
-    trace_metrics: Optional[List[BaseMetric]] = None
-
-    llm_prompt: Optional[Prompt] = None
-    llm_metrics: Optional[List[BaseMetric]] = None
-    llm_metric_collection: Optional[str] = None
-
-    agent_metrics: Optional[List[BaseMetric]] = None
-    agent_metric_collection: Optional[str] = None
+class DeepEvalPydanticAIAgent(Agent):
 
-    def __init__(
-        self,
-        *args,
-        trace_name: Optional[str] = None,
-        trace_tags: Optional[List[str]] = None,
-        trace_metadata: Optional[dict] = None,
-        trace_thread_id: Optional[str] = None,
-        trace_user_id: Optional[str] = None,
-        trace_metric_collection: Optional[str] = None,
-        trace_metrics: Optional[List[BaseMetric]] = None,
-        llm_metric_collection: Optional[str] = None,
-        llm_metrics: Optional[List[BaseMetric]] = None,
-        llm_prompt: Optional[Prompt] = None,
-        agent_metric_collection: Optional[str] = None,
-        agent_metrics: Optional[List[BaseMetric]] = None,
-        **kwargs
-    ):
-        pydantic_ai_installed()
-
-        self.trace_name = trace_name
-        self.trace_tags = trace_tags
-        self.trace_metadata = trace_metadata
-        self.trace_thread_id = trace_thread_id
-        self.trace_user_id = trace_user_id
-        self.trace_metric_collection = trace_metric_collection
-        self.trace_metrics = trace_metrics
-
-        self.llm_metric_collection = llm_metric_collection
-        self.llm_metrics = llm_metrics
-        self.llm_prompt = llm_prompt
-
-        self.agent_metric_collection = agent_metric_collection
-        self.agent_metrics = agent_metrics
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "instrument_pydantic_ai is deprecated and will be removed in a future version. "
+            "Please use the new ConfidentInstrumentationSettings instead. Docs: https://www.confident-ai.com/docs/integrations/third-party/pydantic-ai",
+            DeprecationWarning,
+            stacklevel=2,
+        )
 
         super().__init__(*args, **kwargs)
-
-        patch_llm_model(
-            self._model, llm_metric_collection, llm_metrics, llm_prompt
-        )  # TODO: Add dual patch guards
-
-    async def run(
-        self,
-        *args,
-        name: Optional[str] = None,
-        tags: Optional[List[str]] = None,
-        user_id: Optional[str] = None,
-        metadata: Optional[dict] = None,
-        thread_id: Optional[str] = None,
-        metrics: Optional[List[BaseMetric]] = None,
-        metric_collection: Optional[str] = None,
-        **kwargs
-    ):
-        sig = inspect.signature(super().run)
-        bound = sig.bind_partial(*args, **kwargs)
-        bound.apply_defaults()
-        input = bound.arguments.get("user_prompt", None)
-
-        agent_name = super().name if super().name is not None else "Agent"
-
-        with Observer(
-            span_type="agent" if not _IS_RUN_SYNC.get() else "custom",
-            func_name=agent_name if not _IS_RUN_SYNC.get() else "run",
-            function_kwargs={"input": input},
-            metrics=self.agent_metrics if not _IS_RUN_SYNC.get() else None,
-            metric_collection=(
-                self.agent_metric_collection if not _IS_RUN_SYNC.get() else None
-            ),
-        ) as observer:
-            result = await super().run(*args, **kwargs)
-            observer.result = result.output
-            update_trace_context(
-                trace_name=name if name is not None else self.trace_name,
-                trace_tags=tags if tags is not None else self.trace_tags,
-                trace_metadata=(
-                    metadata if metadata is not None else self.trace_metadata
-                ),
-                trace_thread_id=(
-                    thread_id if thread_id is not None else self.trace_thread_id
-                ),
-                trace_user_id=(
-                    user_id if user_id is not None else self.trace_user_id
-                ),
-                trace_metric_collection=(
-                    metric_collection
-                    if metric_collection is not None
-                    else self.trace_metric_collection
-                ),
-                trace_metrics=(
-                    metrics if metrics is not None else self.trace_metrics
-                ),
-                trace_input=input,
-                trace_output=result.output,
-            )
-
-            agent_span: AgentSpan = current_span_context.get()
-            try:
-                agent_span.tools_called = extract_tools_called(result)
-            except:
-                pass
-            # TODO: available tools
-            # TODO: agent handoffs
-
-        return result
-
-    def run_sync(
-        self,
-        *args,
-        name: Optional[str] = None,
-        tags: Optional[List[str]] = None,
-        metadata: Optional[dict] = None,
-        thread_id: Optional[str] = None,
-        user_id: Optional[str] = None,
-        metric_collection: Optional[str] = None,
-        metrics: Optional[List[BaseMetric]] = None,
-        **kwargs
-    ):
-        sig = inspect.signature(super().run_sync)
-        bound = sig.bind_partial(*args, **kwargs)
-        bound.apply_defaults()
-        input = bound.arguments.get("user_prompt", None)
-
-        token = _IS_RUN_SYNC.set(True)
-
-        agent_name = super().name if super().name is not None else "Agent"
-
-        with Observer(
-            span_type="agent",
-            func_name=agent_name,
-            function_kwargs={"input": input},
-            metrics=self.agent_metrics,
-            metric_collection=self.agent_metric_collection,
-        ) as observer:
-            try:
-                result = super().run_sync(*args, **kwargs)
-            finally:
-                _IS_RUN_SYNC.reset(token)
-
-            observer.result = result.output
-            update_trace_context(
-                trace_name=name if name is not None else self.trace_name,
-                trace_tags=tags if tags is not None else self.trace_tags,
-                trace_metadata=(
-                    metadata if metadata is not None else self.trace_metadata
-                ),
-                trace_thread_id=(
-                    thread_id if thread_id is not None else self.trace_thread_id
-                ),
-                trace_user_id=(
-                    user_id if user_id is not None else self.trace_user_id
-                ),
-                trace_metric_collection=(
-                    metric_collection
-                    if metric_collection is not None
-                    else self.trace_metric_collection
-                ),
-                trace_metrics=(
-                    metrics if metrics is not None else self.trace_metrics
-                ),
-                trace_input=input,
-                trace_output=result.output,
-            )
-
-            agent_span: AgentSpan = current_span_context.get()
-            try:
-                agent_span.tools_called = extract_tools_called(result)
-            except:
-                pass
-
-            # TODO: available tools
-            # TODO: agent handoffs
-
-        return result
-
-    @asynccontextmanager
-    async def run_stream(
-        self,
-        *args,
-        name: Optional[str] = None,
-        tags: Optional[List[str]] = None,
-        metadata: Optional[dict] = None,
-        thread_id: Optional[str] = None,
-        user_id: Optional[str] = None,
-        metric_collection: Optional[str] = None,
-        metrics: Optional[List[BaseMetric]] = None,
-        **kwargs
-    ):
-        sig = inspect.signature(super().run_stream)
-        super_params = sig.parameters
-        super_kwargs = {k: v for k, v in kwargs.items() if k in super_params}
-        bound = sig.bind_partial(*args, **super_kwargs)
-        bound.apply_defaults()
-        input = bound.arguments.get("user_prompt", None)
-
-        agent_name = super().name if super().name is not None else "Agent"
-
-        with Observer(
-            span_type="agent",
-            func_name=agent_name,
-            function_kwargs={"input": input},
-            metrics=self.agent_metrics,
-            metric_collection=self.agent_metric_collection,
-        ) as observer:
-            final_result = None
-            async with super().run_stream(*args, **super_kwargs) as result:
-                try:
-                    yield result
-                finally:
-                    try:
-                        final_result = await result.get_output()
-                        observer.result = final_result
-                    except Exception:
-                        pass
-
-                    update_trace_context(
-                        trace_name=(
-                            name if name is not None else self.trace_name
-                        ),
-                        trace_tags=(
-                            tags if tags is not None else self.trace_tags
-                        ),
-                        trace_metadata=(
-                            metadata
-                            if metadata is not None
-                            else self.trace_metadata
-                        ),
-                        trace_thread_id=(
-                            thread_id
-                            if thread_id is not None
-                            else self.trace_thread_id
-                        ),
-                        trace_user_id=(
-                            user_id
-                            if user_id is not None
-                            else self.trace_user_id
-                        ),
-                        trace_metric_collection=(
-                            metric_collection
-                            if metric_collection is not None
-                            else self.trace_metric_collection
-                        ),
-                        trace_metrics=(
-                            metrics
-                            if metrics is not None
-                            else self.trace_metrics
-                        ),
-                        trace_input=input,
-                        trace_output=(
-                            final_result if final_result is not None else None
-                        ),
-                    )
-                    agent_span: AgentSpan = current_span_context.get()
-                    try:
-                        if final_result is not None:
-                            agent_span.tools_called = extract_tools_called(
-                                final_result
-                            )
-                    except:
-                        pass
-
-    def tool(
-        self,
-        *args,
-        metrics: Optional[List[BaseMetric]] = None,
-        metric_collection: Optional[str] = None,
-        **kwargs
-    ):
-        # Direct decoration: @agent.tool
-        if args and callable(args[0]):
-            patched_func = create_patched_tool(
-                args[0], metrics, metric_collection
-            )
-            new_args = (patched_func,) + args[1:]
-            return super(DeepEvalPydanticAIAgent, self).tool(
-                *new_args, **kwargs
-            )
-        # Decoration with args: @agent.tool(...)
-        super_tool = super(DeepEvalPydanticAIAgent, self).tool
-
-        def decorator(func):
-            patched_func = create_patched_tool(func, metrics, metric_collection)
-            return super_tool(*args, **kwargs)(patched_func)
-
-        return decorator

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -0,0 +1,196 @@
+import json
+import os
+from typing import Literal, Optional, List
+
+try:
+    from pydantic_ai.models.instrumented import InstrumentationSettings
+    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+
+    dependency_installed = True
+except:
+    dependency_installed = False
+
+
+def is_dependency_installed():
+    if not dependency_installed:
+        raise ImportError(
+            "Dependencies are not installed. Please install it with `pip install pydantic-ai opentelemetry-sdk opentelemetry-exporter-otlp-proto-http`."
+        )
+    return True
+
+
+from deepeval.confident.api import get_confident_api_key
+from deepeval.prompt import Prompt
+
+# OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
+OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
+
+
+class SpanInterceptor(SpanProcessor):
+    def __init__(self, settings_instance):
+        # Keep a reference to the settings instance instead of copying values
+        self.settings: ConfidentInstrumentationSettings = settings_instance
+
+    def on_start(self, span, parent_context):
+
+        # set trace attributes
+        if self.settings.thread_id:
+            span.set_attribute(
+                "confident.trace.thread_id", self.settings.thread_id
+            )
+        if self.settings.user_id:
+            span.set_attribute("confident.trace.user_id", self.settings.user_id)
+        if self.settings.metadata:
+            span.set_attribute(
+                "confident.trace.metadata", json.dumps(self.settings.metadata)
+            )
+        if self.settings.tags:
+            span.set_attribute("confident.trace.tags", self.settings.tags)
+        if self.settings.metric_collection:
+            span.set_attribute(
+                "confident.trace.metric_collection",
+                self.settings.metric_collection,
+            )
+        if self.settings.environment:
+            span.set_attribute(
+                "confident.trace.environment", self.settings.environment
+            )
+        if self.settings.name:
+            span.set_attribute("confident.trace.name", self.settings.name)
+        if self.settings.confident_prompt:
+            span.set_attribute(
+                "confident.span.prompt",
+                json.dumps(
+                    {
+                        "alias": self.settings.confident_prompt.alias,
+                        "version": self.settings.confident_prompt.version,
+                    }
+                ),
+            )
+
+        # set trace metric collection
+        if self.settings.trace_metric_collection:
+            span.set_attribute(
+                "confident.trace.metric_collection",
+                self.settings.trace_metric_collection,
+            )
+
+        # set agent name and metric collection
+        if span.attributes.get("agent_name"):
+            span.set_attribute("confident.span.type", "agent")
+            span.set_attribute(
+                "confident.span.name", span.attributes.get("agent_name")
+            )
+            if self.settings.agent_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    self.settings.agent_metric_collection,
+                )
+
+        # set llm metric collection
+        if span.attributes.get("gen_ai.operation.name") in [
+            "chat",
+            "generate_content",
+            "text_completion",
+        ]:
+            if self.settings.llm_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    self.settings.llm_metric_collection,
+                )
+
+        # set tool metric collection
+        tool_name = span.attributes.get("gen_ai.tool.name")
+        if tool_name:
+            tool_metric_collection = (
+                self.settings.tool_metric_collection_map.get(tool_name)
+            )
+            if tool_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    str(tool_metric_collection),
+                )
+
+    def on_end(self, span):
+        pass
+
+
+class ConfidentInstrumentationSettings(InstrumentationSettings):
+
+    name: Optional[str] = None
+    thread_id: Optional[str] = None
+    user_id: Optional[str] = None
+    metadata: Optional[dict] = None
+    tags: Optional[List[str]] = None
+    environment: Literal["production", "staging", "development", "testing"] = (
+        None
+    )
+    metric_collection: Optional[str] = None
+    confident_prompt: Optional[Prompt] = None
+    llm_metric_collection: Optional[str] = None
+    agent_metric_collection: Optional[str] = None
+    tool_metric_collection_map: dict = {}
+    trace_metric_collection: Optional[str] = None
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        name: Optional[str] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metadata: Optional[dict] = None,
+        tags: Optional[List[str]] = None,
+        metric_collection: Optional[str] = None,
+        confident_prompt: Optional[Prompt] = None,
+        llm_metric_collection: Optional[str] = None,
+        agent_metric_collection: Optional[str] = None,
+        tool_metric_collection_map: dict = {},
+        trace_metric_collection: Optional[str] = None,
+    ):
+        is_dependency_installed()
+
+        _environment = os.getenv("CONFIDENT_TRACE_ENVIRONMENT", "development")
+        if _environment and _environment in [
+            "production",
+            "staging",
+            "development",
+            "testing",
+        ]:
+            self.environment = _environment
+
+        self.tool_metric_collection_map = tool_metric_collection_map
+        self.name = name
+        self.thread_id = thread_id
+        self.user_id = user_id
+        self.metadata = metadata
+        self.tags = tags
+        self.metric_collection = metric_collection
+        self.confident_prompt = confident_prompt
+        self.llm_metric_collection = llm_metric_collection
+        self.agent_metric_collection = agent_metric_collection
+        self.trace_metric_collection = trace_metric_collection
+
+        if not api_key:
+            api_key = get_confident_api_key()
+            if not api_key:
+                raise ValueError("CONFIDENT_API_KEY is not set")
+
+        trace_provider = TracerProvider()
+
+        # Pass the entire settings instance instead of individual values
+        span_interceptor = SpanInterceptor(self)
+        trace_provider.add_span_processor(span_interceptor)
+
+        trace_provider.add_span_processor(
+            BatchSpanProcessor(
+                OTLPSpanExporter(
+                    endpoint=OTLP_ENDPOINT,
+                    headers={"x-confident-api-key": api_key},
+                )
+            )
+        )
+        super().__init__(tracer_provider=trace_provider)

deepeval/integrations/pydantic_ai/otel.py

@@ -1,7 +1,6 @@
+import warnings
 from typing import Optional
-import deepeval
 from deepeval.telemetry import capture_tracing_integration
-from deepeval.confident.api import get_confident_api_key
 
 try:
     from opentelemetry import trace
@@ -28,6 +27,13 @@ OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
 
 
 def instrument_pydantic_ai(api_key: Optional[str] = None):
+    warnings.warn(
+        "instrument_pydantic_ai is deprecated and will be removed in a future version. "
+        "Please use the new ConfidentInstrumentationSettings instead. Docs: https://www.confident-ai.com/docs/integrations/third-party/pydantic-ai",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
     with capture_tracing_integration("pydantic_ai"):
         is_opentelemetry_available()