deepeval 3.5.8__tar.gz → 3.6.0__tar.gz

{deepeval-3.5.8 → deepeval-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.8
+Version: 3.6.0
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.6.0/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.6.0"

{deepeval-3.5.8 → deepeval-3.6.0}/deepeval/config/settings_manager.py

@@ -15,7 +15,7 @@ from enum import Enum
 from pydantic import SecretStr
 from deepeval.config.settings import get_settings, _SAVE_RE
 from deepeval.cli.dotenv_handler import DotenvHandler
-from deepeval.utils import bool_to_env_str
+from deepeval.config.utils import bool_to_env_str
 
 logger = logging.getLogger(__name__)
 StrOrEnum = Union[str, Enum]

deepeval-3.6.0/deepeval/contextvars.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Optional
+
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden
+
+
+CURRENT_GOLDEN: ContextVar[Optional[Golden]] = ContextVar(
+    "CURRENT_GOLDEN", default=None
+)
+
+
+def set_current_golden(golden: Optional[Golden]):
+    return CURRENT_GOLDEN.set(golden)
+
+
+def get_current_golden() -> Optional[Golden]:
+    return CURRENT_GOLDEN.get()
+
+
+def reset_current_golden(token) -> None:
+    CURRENT_GOLDEN.reset(token)

deepeval-3.6.0/deepeval/dataset/__init__.py

@@ -0,0 +1,11 @@
+from deepeval.contextvars import get_current_golden
+from .dataset import EvaluationDataset
+from .golden import Golden, ConversationalGolden
+
+
+__all__ = [
+    "EvaluationDataset",
+    "Golden",
+    "ConversationalGolden",
+    "get_current_golden",
+]

{deepeval-3.5.8 → deepeval-3.6.0}/deepeval/evaluate/execute.py

@@ -42,6 +42,7 @@ from deepeval.tracing.api import (
     BaseApiSpan,
 )
 from deepeval.dataset import Golden
+from deepeval.contextvars import set_current_golden, reset_current_golden
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.utils import copy_metrics
 from deepeval.utils import (
@@ -1480,6 +1481,7 @@ def execute_agentic_test_cases_from_loop(
         )
 
         for golden in goldens:
+            token = set_current_golden(golden)
             with capture_evaluation_run("golden"):
                 # yield golden
                 count += 1
@@ -1492,8 +1494,14 @@ def execute_agentic_test_cases_from_loop(
                     _progress=progress,
                     _pbar_callback_id=pbar_tags_id,
                 ):
-                    yield golden
-                    current_trace: Trace = current_trace_context.get()
+                    try:
+                        # yield golden to user code
+                        yield golden
+                        # control has returned from user code without error, capture trace now
+                        current_trace: Trace = current_trace_context.get()
+                    finally:
+                        # after user code returns control, always reset the context
+                        reset_current_golden(token)
 
                 update_pbar(progress, pbar_tags_id)
                 update_pbar(progress, pbar_id)
@@ -1849,6 +1857,7 @@ def a_execute_agentic_test_cases_from_loop(
 
         try:
             for index, golden in enumerate(goldens):
+                token = set_current_golden(golden)
                 current_golden_ctx.update(
                     {
                         "index": index,
@@ -1857,7 +1866,10 @@ def a_execute_agentic_test_cases_from_loop(
                     }
                 )
                 prev_task_length = len(created_tasks)
-                yield golden
+                try:
+                    yield golden
+                finally:
+                    reset_current_golden(token)
                 # if this golden created no tasks, bump bars now
                 if len(created_tasks) == prev_task_length:
                     update_pbar(progress, pbar_callback_id)

deepeval-3.6.0/deepeval/integrations/pydantic_ai/__init__.py

@@ -0,0 +1,5 @@
+from .agent import DeepEvalPydanticAIAgent as Agent
+from .instrumentator import ConfidentInstrumentationSettings
+from .otel import instrument_pydantic_ai
+
+__all__ = ["ConfidentInstrumentationSettings"]

deepeval-3.6.0/deepeval/integrations/pydantic_ai/agent.py

@@ -0,0 +1,21 @@
+import warnings
+
+try:
+    from pydantic_ai.agent import Agent
+
+    is_pydantic_ai_installed = True
+except:
+    is_pydantic_ai_installed = False
+
+
+class DeepEvalPydanticAIAgent(Agent):
+
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "instrument_pydantic_ai is deprecated and will be removed in a future version. "
+            "Please use the new ConfidentInstrumentationSettings instead. Docs: https://www.confident-ai.com/docs/integrations/third-party/pydantic-ai",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        super().__init__(*args, **kwargs)

deepeval-3.6.0/deepeval/integrations/pydantic_ai/instrumentator.py

@@ -0,0 +1,196 @@
+import json
+import os
+from typing import Literal, Optional, List
+
+try:
+    from pydantic_ai.models.instrumented import InstrumentationSettings
+    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+
+    dependency_installed = True
+except:
+    dependency_installed = False
+
+
+def is_dependency_installed():
+    if not dependency_installed:
+        raise ImportError(
+            "Dependencies are not installed. Please install it with `pip install pydantic-ai opentelemetry-sdk opentelemetry-exporter-otlp-proto-http`."
+        )
+    return True
+
+
+from deepeval.confident.api import get_confident_api_key
+from deepeval.prompt import Prompt
+
+# OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
+OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
+
+
+class SpanInterceptor(SpanProcessor):
+    def __init__(self, settings_instance):
+        # Keep a reference to the settings instance instead of copying values
+        self.settings: ConfidentInstrumentationSettings = settings_instance
+
+    def on_start(self, span, parent_context):
+
+        # set trace attributes
+        if self.settings.thread_id:
+            span.set_attribute(
+                "confident.trace.thread_id", self.settings.thread_id
+            )
+        if self.settings.user_id:
+            span.set_attribute("confident.trace.user_id", self.settings.user_id)
+        if self.settings.metadata:
+            span.set_attribute(
+                "confident.trace.metadata", json.dumps(self.settings.metadata)
+            )
+        if self.settings.tags:
+            span.set_attribute("confident.trace.tags", self.settings.tags)
+        if self.settings.metric_collection:
+            span.set_attribute(
+                "confident.trace.metric_collection",
+                self.settings.metric_collection,
+            )
+        if self.settings.environment:
+            span.set_attribute(
+                "confident.trace.environment", self.settings.environment
+            )
+        if self.settings.name:
+            span.set_attribute("confident.trace.name", self.settings.name)
+        if self.settings.confident_prompt:
+            span.set_attribute(
+                "confident.span.prompt",
+                json.dumps(
+                    {
+                        "alias": self.settings.confident_prompt.alias,
+                        "version": self.settings.confident_prompt.version,
+                    }
+                ),
+            )
+
+        # set trace metric collection
+        if self.settings.trace_metric_collection:
+            span.set_attribute(
+                "confident.trace.metric_collection",
+                self.settings.trace_metric_collection,
+            )
+
+        # set agent name and metric collection
+        if span.attributes.get("agent_name"):
+            span.set_attribute("confident.span.type", "agent")
+            span.set_attribute(
+                "confident.span.name", span.attributes.get("agent_name")
+            )
+            if self.settings.agent_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    self.settings.agent_metric_collection,
+                )
+
+        # set llm metric collection
+        if span.attributes.get("gen_ai.operation.name") in [
+            "chat",
+            "generate_content",
+            "text_completion",
+        ]:
+            if self.settings.llm_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    self.settings.llm_metric_collection,
+                )
+
+        # set tool metric collection
+        tool_name = span.attributes.get("gen_ai.tool.name")
+        if tool_name:
+            tool_metric_collection = (
+                self.settings.tool_metric_collection_map.get(tool_name)
+            )
+            if tool_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    str(tool_metric_collection),
+                )
+
+    def on_end(self, span):
+        pass
+
+
+class ConfidentInstrumentationSettings(InstrumentationSettings):
+
+    name: Optional[str] = None
+    thread_id: Optional[str] = None
+    user_id: Optional[str] = None
+    metadata: Optional[dict] = None
+    tags: Optional[List[str]] = None
+    environment: Literal["production", "staging", "development", "testing"] = (
+        None
+    )
+    metric_collection: Optional[str] = None
+    confident_prompt: Optional[Prompt] = None
+    llm_metric_collection: Optional[str] = None
+    agent_metric_collection: Optional[str] = None
+    tool_metric_collection_map: dict = {}
+    trace_metric_collection: Optional[str] = None
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        name: Optional[str] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metadata: Optional[dict] = None,
+        tags: Optional[List[str]] = None,
+        metric_collection: Optional[str] = None,
+        confident_prompt: Optional[Prompt] = None,
+        llm_metric_collection: Optional[str] = None,
+        agent_metric_collection: Optional[str] = None,
+        tool_metric_collection_map: dict = {},
+        trace_metric_collection: Optional[str] = None,
+    ):
+        is_dependency_installed()
+
+        _environment = os.getenv("CONFIDENT_TRACE_ENVIRONMENT", "development")
+        if _environment and _environment in [
+            "production",
+            "staging",
+            "development",
+            "testing",
+        ]:
+            self.environment = _environment
+
+        self.tool_metric_collection_map = tool_metric_collection_map
+        self.name = name
+        self.thread_id = thread_id
+        self.user_id = user_id
+        self.metadata = metadata
+        self.tags = tags
+        self.metric_collection = metric_collection
+        self.confident_prompt = confident_prompt
+        self.llm_metric_collection = llm_metric_collection
+        self.agent_metric_collection = agent_metric_collection
+        self.trace_metric_collection = trace_metric_collection
+
+        if not api_key:
+            api_key = get_confident_api_key()
+            if not api_key:
+                raise ValueError("CONFIDENT_API_KEY is not set")
+
+        trace_provider = TracerProvider()
+
+        # Pass the entire settings instance instead of individual values
+        span_interceptor = SpanInterceptor(self)
+        trace_provider.add_span_processor(span_interceptor)
+
+        trace_provider.add_span_processor(
+            BatchSpanProcessor(
+                OTLPSpanExporter(
+                    endpoint=OTLP_ENDPOINT,
+                    headers={"x-confident-api-key": api_key},
+                )
+            )
+        )
+        super().__init__(tracer_provider=trace_provider)

{deepeval-3.5.8 → deepeval-3.6.0}/deepeval/integrations/pydantic_ai/otel.py

@@ -1,7 +1,6 @@
+import warnings
 from typing import Optional
-import deepeval
 from deepeval.telemetry import capture_tracing_integration
-from deepeval.confident.api import get_confident_api_key
 
 try:
     from opentelemetry import trace
@@ -28,6 +27,13 @@ OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
 
 
 def instrument_pydantic_ai(api_key: Optional[str] = None):
+    warnings.warn(
+        "instrument_pydantic_ai is deprecated and will be removed in a future version. "
+        "Please use the new ConfidentInstrumentationSettings instead. Docs: https://www.confident-ai.com/docs/integrations/third-party/pydantic-ai",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
     with capture_tracing_integration("pydantic_ai"):
         is_opentelemetry_available()
 

{deepeval-3.5.8 → deepeval-3.6.0}/deepeval/openai_agents/__init__.py

@@ -1,6 +1,7 @@
 from deepeval.openai_agents.callback_handler import DeepEvalTracingProcessor
-from deepeval.openai_agents.runner import Runner
-from deepeval.openai_agents.patch import function_tool
 from deepeval.openai_agents.agent import DeepEvalAgent as Agent
+from deepeval.openai_agents.patch import function_tool
+
+# from deepeval.openai_agents.runner import Runner
 
-__all__ = ["DeepEvalTracingProcessor", "Runner", "function_tool", "Agent"]
+__all__ = ["DeepEvalTracingProcessor", "Agent", "function_tool"]

deepeval-3.6.0/deepeval/openai_agents/agent.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Generic, TypeVar, List
+
+from deepeval.prompt import Prompt
+from deepeval.metrics import BaseMetric
+from deepeval.tracing.types import LlmSpan
+
+try:
+    from agents.agent import Agent as BaseAgent
+    from deepeval.openai_agents.patch import (
+        patch_default_agent_runner_get_model,
+    )
+except Exception as e:
+    raise RuntimeError(
+        "openai-agents is required for this integration. Please install it."
+    ) from e
+
+TContext = TypeVar("TContext")
+
+
+@dataclass
+class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
+    """
+    A subclass of agents.Agent.
+    """
+
+    llm_metric_collection: str = None
+    llm_metrics: List[BaseMetric] = None
+    confident_prompt: Prompt = None
+    agent_metrics: List[BaseMetric] = None
+    agent_metric_collection: str = None
+
+    def __post_init__(self):
+        patch_default_agent_runner_get_model()

deepeval-3.6.0/deepeval/openai_agents/callback_handler.py

@@ -0,0 +1,135 @@
+from deepeval.tracing.tracing import (
+    Observer,
+    current_span_context,
+    trace_manager,
+)
+from deepeval.openai_agents.extractors import *
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.utils import make_json_serializable
+from time import perf_counter
+from deepeval.tracing.types import TraceSpanStatus
+
+try:
+    from agents.tracing import Span, Trace, TracingProcessor
+    from agents.tracing.span_data import (
+        AgentSpanData,
+        CustomSpanData,
+        FunctionSpanData,
+        GenerationSpanData,
+        GuardrailSpanData,
+        HandoffSpanData,
+        ResponseSpanData,
+        SpanData,
+    )
+    from deepeval.openai_agents.patch import (
+        patch_default_agent_run_single_turn,
+        patch_default_agent_run_single_turn_streamed,
+    )
+
+    openai_agents_available = True
+except ImportError:
+    openai_agents_available = False
+
+
+def _check_openai_agents_available():
+    if not openai_agents_available:
+        raise ImportError(
+            "openai-agents is required for this integration. Install it via your package manager"
+        )
+
+
+class DeepEvalTracingProcessor(TracingProcessor):
+    def __init__(self) -> None:
+        _check_openai_agents_available()
+        patch_default_agent_run_single_turn()
+        patch_default_agent_run_single_turn_streamed()
+        self.span_observers: dict[str, Observer] = {}
+
+    def on_trace_start(self, trace: "Trace") -> None:
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _thread_id = trace_dict.get("group_id")
+        _trace_name = trace_dict.get("workflow_name")
+        _trace_metadata = trace_dict.get("metadata")
+
+        _trace = trace_manager.start_new_trace(trace_uuid=str(_trace_uuid))
+        _trace.thread_id = str(_thread_id)
+        _trace.name = str(_trace_name)
+        _trace.metadata = make_json_serializable(_trace_metadata)
+        current_trace_context.set(_trace)
+
+        trace_manager.add_span(  # adds a dummy root span
+            BaseSpan(
+                uuid=_trace_uuid,
+                trace_uuid=_trace_uuid,
+                parent_uuid=None,
+                start_time=perf_counter(),
+                name=_trace_name,
+                status=TraceSpanStatus.IN_PROGRESS,
+                children=[],
+            )
+        )
+
+    def on_trace_end(self, trace: "Trace") -> None:
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _trace_name = trace_dict.get("workflow_name")
+
+        trace_manager.remove_span(_trace_uuid)  # removing the dummy root span
+        trace_manager.end_trace(_trace_uuid)
+        current_trace_context.set(None)
+
+    def on_span_start(self, span: "Span") -> None:
+        if not span.started_at:
+            return
+        current_span = current_span_context.get()
+        if current_span and isinstance(current_span, LlmSpan):
+            return
+
+        span_type = self.get_span_kind(span.span_data)
+        observer = Observer(span_type=span_type, func_name="NA")
+        if span_type == "llm":
+            observer.observe_kwargs["model"] = "temporary model"
+        observer.update_span_properties = (
+            lambda span_type: update_span_properties(span_type, span.span_data)
+        )
+        self.span_observers[span.span_id] = observer
+        observer.__enter__()
+
+    def on_span_end(self, span: "Span") -> None:
+        update_trace_properties_from_span_data(
+            current_trace_context.get(), span.span_data
+        )
+
+        current_span = current_span_context.get()
+        if current_span and isinstance(current_span, LlmSpan):
+            update_span_properties(current_span, span.span_data)
+            return
+        observer = self.span_observers.pop(span.span_id, None)
+        if observer:
+            observer.__exit__(None, None, None)
+
+    def force_flush(self) -> None:
+        pass
+
+    def shutdown(self) -> None:
+        pass
+
+    def get_span_kind(self, span_data: "SpanData") -> str:
+        if isinstance(span_data, AgentSpanData):
+            return "agent"
+        if isinstance(span_data, FunctionSpanData):
+            return "tool"
+        if isinstance(span_data, MCPListToolsSpanData):
+            return "tool"
+        if isinstance(span_data, GenerationSpanData):
+            return "llm"
+        if isinstance(span_data, ResponseSpanData):
+            return "llm"
+        if isinstance(span_data, HandoffSpanData):
+            return "custom"
+        if isinstance(span_data, CustomSpanData):
+            return "base"
+        if isinstance(span_data, GuardrailSpanData):
+            return "base"
+        return "base"