deepeval 3.8.1__py3-none-any.whl → 3.8.3__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.8.1"
+__version__: str = "3.8.3"

deepeval/confident/api.py

@@ -26,16 +26,44 @@ API_BASE_URL_EU = "https://eu.api.confident-ai.com"
 retryable_exceptions = requests.exceptions.SSLError
 
 
+def _infer_region_from_api_key(api_key: Optional[str]) -> Optional[str]:
+    """
+    Infer region from Confident API key prefix.
+
+    Supported:
+      - confident_eu_... => "EU"
+      - confident_us_... => "US"
+
+    Returns None if prefix is not recognized or api_key is falsy.
+    """
+    if not api_key:
+        return None
+    key = api_key.strip().lower()
+    if key.startswith("confident_eu_"):
+        return "EU"
+    if key.startswith("confident_us_"):
+        return "US"
+    return None
+
+
 def get_base_api_url():
     s = get_settings()
     if s.CONFIDENT_BASE_URL:
         base_url = s.CONFIDENT_BASE_URL.rstrip("/")
         return base_url
+    # If the user has explicitly set a region, respect it.
     region = KEY_FILE_HANDLER.fetch_data(KeyValues.CONFIDENT_REGION)
-    if region == "EU":
+    if region:
+        return API_BASE_URL_EU if region == "EU" else API_BASE_URL
+
+    # Otherwise, infer region from the API key prefix.
+    api_key = get_confident_api_key()
+    inferred = _infer_region_from_api_key(api_key)
+    if inferred == "EU":
         return API_BASE_URL_EU
-    else:
-        return API_BASE_URL
+
+    # Default to US (backwards compatible)
+    return API_BASE_URL
 
 
 def get_confident_api_key() -> Optional[str]:

deepeval/integrations/crewai/__init__.py

@@ -1,4 +1,4 @@
-from .handler import instrument_crewai
+from .handler import instrument_crewai, reset_crewai_instrumentation
 from .subs import (
     DeepEvalCrew as Crew,
     DeepEvalAgent as Agent,
@@ -6,4 +6,11 @@ from .subs import (
 )
 from .tool import tool
 
-__all__ = ["instrument_crewai", "Crew", "Agent", "LLM", "tool"]
+__all__ = [
+    "instrument_crewai",
+    "Crew",
+    "Agent",
+    "LLM",
+    "tool",
+    "reset_crewai_instrumentation",
+]

deepeval/integrations/crewai/handler.py

@@ -1,11 +1,12 @@
 import logging
 import deepeval
-
-from typing import Optional
+from collections import defaultdict
+from time import perf_counter
+from typing import Optional, Tuple, Any, List, Union
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing.context import current_span_context, current_trace_context
-from deepeval.tracing.tracing import Observer
-from deepeval.tracing.types import LlmSpan
+from deepeval.tracing.tracing import Observer, trace_manager
+from deepeval.tracing.types import ToolSpan, SpanType, TraceSpanStatus
 from deepeval.config.settings import get_settings
 
 
@@ -45,7 +46,9 @@ except ImportError as e:
 
     crewai_installed = False
 
+# GLOBAL STATE to prevent duplicate listeners
 IS_WRAPPED_ALL = False
+_listener_instance = None
 
 
 def is_crewai_installed():
@@ -55,21 +58,49 @@ def is_crewai_installed():
         )
 
 
+def _get_metrics_data(obj: Any) -> Tuple[Optional[str], Optional[Any]]:
+    """Helper to safely extract metrics attached to CrewAI objects."""
+
+    if not obj:
+        return None, None
+    metric_collection = getattr(obj, "_metric_collection", None)
+    metrics = getattr(obj, "_metrics", None)
+
+    if metric_collection is not None or metrics is not None:
+        return metric_collection, metrics
+
+    func = getattr(obj, "func", None)
+    if func:
+        metric_collection = getattr(func, "_metric_collection", None)
+        metrics = getattr(func, "_metrics", None)
+
+    return metric_collection, metrics
+
+
 class CrewAIEventsListener(BaseEventListener):
     def __init__(self):
         is_crewai_installed()
         super().__init__()
         self.span_observers: dict[str, Observer] = {}
+        self.tool_observers_stack: dict[str, List[Union[Observer, None]]] = (
+            defaultdict(list)
+        )
 
-    @staticmethod
-    def get_tool_execution_id(source, event) -> str:
-        source_id = id(source)
-        task_id = getattr(event, "task_id", "unknown")
-        agent_id = getattr(event, "agent_id", "unknown")
-        tool_name = getattr(event, "tool_name", "unknown")
-        execution_id = f"tool_{source_id}_{task_id}_{agent_id}_{tool_name}"
+    def reset_state(self):
+        """Clears all internal state to prevent pollution between tests."""
+        self.span_observers.clear()
+        self.tool_observers_stack.clear()
 
-        return execution_id
+    @staticmethod
+    def get_tool_stack_key(source, tool_name) -> str:
+        """
+        Generates a unique key for the tool stack.
+        FIX: Uses role/name instead of id() to be robust against object copying by CrewAI.
+        """
+        identifier = getattr(
+            source, "role", getattr(source, "name", str(id(source)))
+        )
+        return f"{tool_name}_{identifier}"
 
     @staticmethod
     def get_knowledge_execution_id(source, event) -> str:
@@ -79,98 +110,234 @@ class CrewAIEventsListener(BaseEventListener):
 
         return execution_id
 
+    @staticmethod
+    def get_llm_execution_id(source, event) -> str:
+        source_id = id(source)
+        return f"llm_{source_id}"
+
+    def _flatten_tool_span(self, span):
+        """
+        Callback to move any child ToolSpans up to the parent.
+        """
+        if not span.parent_uuid or not span.children:
+            return
+
+        parent_span = trace_manager.get_span_by_uuid(span.parent_uuid)
+        if not parent_span:
+            return
+
+        # Identify child tool spans (ghost nesting)
+        tools_to_move = [
+            child for child in span.children if isinstance(child, ToolSpan)
+        ]
+
+        if tools_to_move:
+            if parent_span.children is None:
+                parent_span.children = []
+
+            for child in tools_to_move:
+                child.parent_uuid = parent_span.uuid
+                parent_span.children.append(child)
+
+            span.children = [
+                child
+                for child in span.children
+                if not isinstance(child, ToolSpan)
+            ]
+
     def setup_listeners(self, crewai_event_bus):
         @crewai_event_bus.on(CrewKickoffStartedEvent)
         def on_crew_started(source, event: CrewKickoffStartedEvent):
-            # Assuming that this event is called in the crew.kickoff method
             current_span = current_span_context.get()
-
-            # set the input
             if current_span:
                 current_span.input = event.inputs
-
-            # set trace input
             current_trace = current_trace_context.get()
             if current_trace:
                 current_trace.input = event.inputs
 
         @crewai_event_bus.on(CrewKickoffCompletedEvent)
         def on_crew_completed(source, event: CrewKickoffCompletedEvent):
-            # Assuming that this event is called in the crew.kickoff method
             current_span = current_span_context.get()
-
-            # set the output
+            output = getattr(
+                event, "output", getattr(event, "result", str(event))
+            )
             if current_span:
-                current_span.output = str(event.output)
-
-            # set trace output
+                current_span.output = str(output)
             current_trace = current_trace_context.get()
             if current_trace:
-                current_trace.output = str(event.output)
+                current_trace.output = str(output)
 
         @crewai_event_bus.on(LLMCallStartedEvent)
         def on_llm_started(source, event: LLMCallStartedEvent):
-            # Assuming that this event is called in the llm.call method
-            current_span = current_span_context.get()
-
-            # set the input
-            if current_span:
-                current_span.input = event.messages
+            metric_collection, metrics = _get_metrics_data(source)
+            observer = Observer(
+                span_type="llm",
+                func_name="call",
+                observe_kwargs={"model": getattr(event, "model", "unknown")},
+                metric_collection=metric_collection,
+                metrics=metrics,
+            )
+            self.span_observers[self.get_llm_execution_id(source, event)] = (
+                observer
+            )
+            observer.__enter__()
 
-                # set the model
-                if isinstance(current_span, LlmSpan):
-                    current_span.model = event.model
+            if observer.trace_uuid:
+                span = trace_manager.get_span_by_uuid(observer.uuid)
+                if span:
+                    msgs = getattr(event, "messages")
+                    span.input = msgs
 
         @crewai_event_bus.on(LLMCallCompletedEvent)
         def on_llm_completed(source, event: LLMCallCompletedEvent):
-            # Assuming that this event is called in the llm.call method
-            current_span = current_span_context.get()
-
-            # set the output
-            if current_span:
-                current_span.output = event.response
+            key = self.get_llm_execution_id(source, event)
+            if key in self.span_observers:
+                observer = self.span_observers.pop(key)
+                if observer:
+                    current_span = current_span_context.get()
+                    token = None
+                    span_to_close = trace_manager.get_span_by_uuid(
+                        observer.uuid
+                    )
+
+                    if span_to_close:
+                        output = getattr(
+                            event, "response", getattr(event, "output", "")
+                        )
+                        span_to_close.output = output
+                        if (
+                            not current_span
+                            or current_span.uuid != observer.uuid
+                        ):
+                            token = current_span_context.set(span_to_close)
+
+                    observer.__exit__(None, None, None)
+                    if token:
+                        current_span_context.reset(token)
 
         @crewai_event_bus.on(AgentExecutionStartedEvent)
         def on_agent_started(source, event: AgentExecutionStartedEvent):
-            # Assuming that this event is called in the agent.execute_task method
             current_span = current_span_context.get()
-
-            # set the input
             if current_span:
                 current_span.input = event.task_prompt
 
         @crewai_event_bus.on(AgentExecutionCompletedEvent)
         def on_agent_completed(source, event: AgentExecutionCompletedEvent):
-            # Assuming that this event is called in the agent.execute_task method
             current_span = current_span_context.get()
-
-            # set the output
             if current_span:
-                current_span.output = event.output
+                current_span.output = getattr(
+                    event, "output", getattr(event, "result", "")
+                )
 
         @crewai_event_bus.on(ToolUsageStartedEvent)
         def on_tool_started(source, event: ToolUsageStartedEvent):
+            key = self.get_tool_stack_key(source, event.tool_name)
+
+            # 1. Internal Stack Check
+            if self.tool_observers_stack[key]:
+                self.tool_observers_stack[key].append(None)
+                return
+
+            # 2. SMART DEDUPING
+            current_span = current_span_context.get()
+            span_type = getattr(current_span, "type", None)
+            is_tool_span = span_type == "tool" or span_type == SpanType.TOOL
+            if (
+                is_tool_span
+                and getattr(current_span, "name", "") == event.tool_name
+            ):
+                self.tool_observers_stack[key].append(None)
+                return
+
+            metric_collection = None
+            metrics = None
+
+            if hasattr(source, "tools"):
+                for tools in source.tools:
+                    if getattr(tools, "name", None) == event.tool_name:
+                        metric_collection, metrics = _get_metrics_data(tools)
+                        break
+
+            if not metric_collection:
+                agent = getattr(source, "agent", source)
+                metric_collection, metrics = _get_metrics_data(agent)
+
             observer = Observer(
                 span_type="tool",
                 func_name=event.tool_name,
                 function_kwargs=event.tool_args,
+                metric_collection=metric_collection,
+                metrics=metrics,
             )
-            self.span_observers[self.get_tool_execution_id(source, event)] = (
-                observer
-            )
+
+            self.tool_observers_stack[key].append(observer)
             observer.__enter__()
 
         @crewai_event_bus.on(ToolUsageFinishedEvent)
         def on_tool_completed(source, event: ToolUsageFinishedEvent):
-            observer = self.span_observers.pop(
-                self.get_tool_execution_id(source, event)
-            )
+            key = self.get_tool_stack_key(source, event.tool_name)
+            observer = None
+
+            if (
+                key in self.tool_observers_stack
+                and self.tool_observers_stack[key]
+            ):
+                item = self.tool_observers_stack[key].pop()
+                if item is None:
+                    return
+                observer = item
+
+            if not observer:
+                current_span = current_span_context.get()
+                if (
+                    current_span
+                    and getattr(current_span, "type", None)
+                    in ["tool", SpanType.TOOL]
+                    and getattr(current_span, "name", "") == event.tool_name
+                ):
+                    current_span.output = getattr(
+                        event, "output", getattr(event, "result", None)
+                    )
+
+                    if current_span.end_time is None:
+                        current_span.end_time = perf_counter()
+
+                    current_span.status = TraceSpanStatus.SUCCESS
+
+                    self._flatten_tool_span(current_span)
+                    trace_manager.remove_span(current_span.uuid)
+
+                    if current_span.parent_uuid:
+                        parent = trace_manager.get_span_by_uuid(
+                            current_span.parent_uuid
+                        )
+                        current_span_context.set(parent if parent else None)
+                    else:
+                        current_span_context.set(None)
+                    return
+
             if observer:
                 current_span = current_span_context.get()
-                if current_span:
-                    current_span.output = event.output
+                token = None
+                span_to_close = trace_manager.get_span_by_uuid(observer.uuid)
+
+                if span_to_close:
+                    span_to_close.output = getattr(
+                        event, "output", getattr(event, "result", None)
+                    )
+                    if not current_span or current_span.uuid != observer.uuid:
+                        token = current_span_context.set(span_to_close)
+
+                observer.update_span_properties = self._flatten_tool_span
                 observer.__exit__(None, None, None)
 
+                if span_to_close and span_to_close.end_time is None:
+                    span_to_close.end_time = perf_counter()
+                    span_to_close.status = TraceSpanStatus.SUCCESS
+
+                if token:
+                    current_span_context.reset(token)
+
         @crewai_event_bus.on(KnowledgeRetrievalStartedEvent)
         def on_knowledge_started(source, event: KnowledgeRetrievalStartedEvent):
             observer = Observer(
@@ -187,18 +354,35 @@ class CrewAIEventsListener(BaseEventListener):
         def on_knowledge_completed(
             source, event: KnowledgeRetrievalCompletedEvent
         ):
-            observer = self.span_observers.pop(
-                self.get_knowledge_execution_id(source, event)
-            )
-            if observer:
-                current_span = current_span_context.get()
-                if current_span:
-                    current_span.input = event.query
-                    current_span.output = event.retrieved_knowledge
-                observer.__exit__(None, None, None)
+            key = self.get_knowledge_execution_id(source, event)
+            if key in self.span_observers:
+                observer = self.span_observers.pop(key)
+                if observer:
+                    current_span = current_span_context.get()
+                    token = None
+                    span_to_close = trace_manager.get_span_by_uuid(
+                        observer.uuid
+                    )
+
+                    if span_to_close:
+                        span_to_close.input = event.query
+                        span_to_close.output = event.retrieved_knowledge
+
+                        if (
+                            not current_span
+                            or current_span.uuid != observer.uuid
+                        ):
+                            token = current_span_context.set(span_to_close)
+
+                    observer.__exit__(None, None, None)
+
+                    if token:
+                        current_span_context.reset(token)
 
 
 def instrument_crewai(api_key: Optional[str] = None):
+    global _listener_instance
+
     is_crewai_installed()
     with capture_tracing_integration("crewai"):
         if api_key:
@@ -206,7 +390,14 @@ def instrument_crewai(api_key: Optional[str] = None):
 
         wrap_all()
 
-        CrewAIEventsListener()
+        if _listener_instance is None:
+            _listener_instance = CrewAIEventsListener()
+
+
+def reset_crewai_instrumentation():
+    global _listener_instance
+    if _listener_instance:
+        _listener_instance.reset_state()
 
 
 def wrap_all():
@@ -218,15 +409,19 @@ def wrap_all():
             wrap_crew_kickoff_for_each,
             wrap_crew_kickoff_async,
             wrap_crew_kickoff_for_each_async,
-            wrap_llm_call,
+            wrap_crew_akickoff,
+            wrap_crew_akickoff_for_each,
             wrap_agent_execute_task,
+            wrap_agent_aexecute_task,
         )
 
         wrap_crew_kickoff()
         wrap_crew_kickoff_for_each()
         wrap_crew_kickoff_async()
         wrap_crew_kickoff_for_each_async()
-        wrap_llm_call()
+        wrap_crew_akickoff()
+        wrap_crew_akickoff_for_each()
         wrap_agent_execute_task()
+        wrap_agent_aexecute_task()
 
         IS_WRAPPED_ALL = True

deepeval/integrations/crewai/subs.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Type, TypeVar
+from typing import List, Optional, Type, TypeVar, Callable
 from pydantic import PrivateAttr
 
 from deepeval.metrics.base_metric import BaseMetric
@@ -28,14 +28,10 @@ def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
         _metric_collection: Optional[str] = PrivateAttr(default=None)
         _metrics: Optional[List[BaseMetric]] = PrivateAttr(default=None)
 
-        def __init__(
-            self,
-            *args,
-            metrics: Optional[List[BaseMetric]] = None,
-            metric_collection: Optional[str] = None,
-            **kwargs
-        ):
+        def __init__(self, *args, **kwargs):
             is_crewai_installed()
+            metric_collection = kwargs.pop("metric_collection", None)
+            metrics = kwargs.pop("metrics", None)
             super().__init__(*args, **kwargs)
             self._metric_collection = metric_collection
             self._metrics = metrics
@@ -45,7 +41,24 @@ def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
     return DeepEvalClass
 
 
-# Create the classes
+def create_deepeval_llm(base_factory: Callable) -> Callable:
+    """Wrapper for factory functions/classes (LLM)."""
+
+    def factory_wrapper(*args, **kwargs):
+        is_crewai_installed()
+        metric_collection = kwargs.pop("metric_collection", None)
+        metrics = kwargs.pop("metrics", None)
+        instance = base_factory(*args, **kwargs)
+        try:
+            instance._metric_collection = metric_collection
+            instance._metrics = metrics
+        except Exception:
+            pass
+        return instance
+
+    return factory_wrapper
+
+
 DeepEvalCrew = create_deepeval_class(Crew, "DeepEvalCrew")
 DeepEvalAgent = create_deepeval_class(Agent, "DeepEvalAgent")
-DeepEvalLLM = create_deepeval_class(LLM, "DeepEvalLLM")
+DeepEvalLLM = create_deepeval_llm(LLM)

deepeval/integrations/crewai/tool.py

@@ -15,6 +15,20 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
     """
     crewai_kwargs = kwargs
 
+    def _attach_metadata(tool_instance):
+        try:
+            object.__setattr__(
+                tool_instance, "metric_collection", metric_collection
+            )
+            object.__setattr__(tool_instance, "metrics", metric)
+        except Exception:
+            try:
+                tool_instance._metric_collection = metric_collection
+                tool_instance._metrics = metric
+            except Exception:
+                pass
+        return tool_instance
+
     # Case 1: @tool (function passed directly)
     if len(args) == 1 and callable(args[0]):
         f = args[0]
@@ -29,7 +43,8 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
             result = f(*f_args, **f_kwargs)
             return result
 
-        return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+        tool_instance = crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+        return _attach_metadata(tool_instance)
 
     # Case 2: @tool("name")
     if len(args) == 1 and isinstance(args[0], str):
@@ -45,7 +60,8 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
                 result = f(*f_args, **f_kwargs)
                 return result
 
-            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            tool_instance = crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            return _attach_metadata(tool_instance)
 
         return _decorator
 
@@ -64,7 +80,8 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
                 result = f(*f_args, **f_kwargs)
                 return result
 
-            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            tool_instance = crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            return _attach_metadata(tool_instance)
 
         return _decorator