deepeval 3.8.2__py3-none-any.whl → 3.8.3__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.8.2"
+__version__: str = "3.8.3"

deepeval/integrations/crewai/__init__.py

@@ -1,4 +1,4 @@
-from .handler import instrument_crewai
+from .handler import instrument_crewai, reset_crewai_instrumentation
 from .subs import (
     DeepEvalCrew as Crew,
     DeepEvalAgent as Agent,
@@ -6,4 +6,11 @@ from .subs import (
 )
 from .tool import tool
 
-__all__ = ["instrument_crewai", "Crew", "Agent", "LLM", "tool"]
+__all__ = [
+    "instrument_crewai",
+    "Crew",
+    "Agent",
+    "LLM",
+    "tool",
+    "reset_crewai_instrumentation",
+]

deepeval/integrations/crewai/handler.py

@@ -1,11 +1,12 @@
 import logging
 import deepeval
-
-from typing import Optional
+from collections import defaultdict
+from time import perf_counter
+from typing import Optional, Tuple, Any, List, Union
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing.context import current_span_context, current_trace_context
-from deepeval.tracing.tracing import Observer
-from deepeval.tracing.types import LlmSpan
+from deepeval.tracing.tracing import Observer, trace_manager
+from deepeval.tracing.types import ToolSpan, SpanType, TraceSpanStatus
 from deepeval.config.settings import get_settings
 
 
@@ -45,7 +46,9 @@ except ImportError as e:
 
     crewai_installed = False
 
+# GLOBAL STATE to prevent duplicate listeners
 IS_WRAPPED_ALL = False
+_listener_instance = None
 
 
 def is_crewai_installed():
@@ -55,21 +58,49 @@ def is_crewai_installed():
         )
 
 
+def _get_metrics_data(obj: Any) -> Tuple[Optional[str], Optional[Any]]:
+    """Helper to safely extract metrics attached to CrewAI objects."""
+
+    if not obj:
+        return None, None
+    metric_collection = getattr(obj, "_metric_collection", None)
+    metrics = getattr(obj, "_metrics", None)
+
+    if metric_collection is not None or metrics is not None:
+        return metric_collection, metrics
+
+    func = getattr(obj, "func", None)
+    if func:
+        metric_collection = getattr(func, "_metric_collection", None)
+        metrics = getattr(func, "_metrics", None)
+
+    return metric_collection, metrics
+
+
 class CrewAIEventsListener(BaseEventListener):
     def __init__(self):
         is_crewai_installed()
         super().__init__()
         self.span_observers: dict[str, Observer] = {}
+        self.tool_observers_stack: dict[str, List[Union[Observer, None]]] = (
+            defaultdict(list)
+        )
 
-    @staticmethod
-    def get_tool_execution_id(source, event) -> str:
-        source_id = id(source)
-        task_id = getattr(event, "task_id", "unknown")
-        agent_id = getattr(event, "agent_id", "unknown")
-        tool_name = getattr(event, "tool_name", "unknown")
-        execution_id = f"tool_{source_id}_{task_id}_{agent_id}_{tool_name}"
+    def reset_state(self):
+        """Clears all internal state to prevent pollution between tests."""
+        self.span_observers.clear()
+        self.tool_observers_stack.clear()
 
-        return execution_id
+    @staticmethod
+    def get_tool_stack_key(source, tool_name) -> str:
+        """
+        Generates a unique key for the tool stack.
+        FIX: Uses role/name instead of id() to be robust against object copying by CrewAI.
+        """
+        identifier = getattr(
+            source, "role", getattr(source, "name", str(id(source)))
+        )
+        return f"{tool_name}_{identifier}"
 
     @staticmethod
     def get_knowledge_execution_id(source, event) -> str:
@@ -79,98 +110,234 @@ class CrewAIEventsListener(BaseEventListener):
 
         return execution_id
 
+    @staticmethod
+    def get_llm_execution_id(source, event) -> str:
+        source_id = id(source)
+        return f"llm_{source_id}"
+
+    def _flatten_tool_span(self, span):
+        """
+        Callback to move any child ToolSpans up to the parent.
+        """
+        if not span.parent_uuid or not span.children:
+            return
+
+        parent_span = trace_manager.get_span_by_uuid(span.parent_uuid)
+        if not parent_span:
+            return
+
+        # Identify child tool spans (ghost nesting)
+        tools_to_move = [
+            child for child in span.children if isinstance(child, ToolSpan)
+        ]
+
+        if tools_to_move:
+            if parent_span.children is None:
+                parent_span.children = []
+
+            for child in tools_to_move:
+                child.parent_uuid = parent_span.uuid
+                parent_span.children.append(child)
+
+            span.children = [
+                child
+                for child in span.children
+                if not isinstance(child, ToolSpan)
+            ]
+
     def setup_listeners(self, crewai_event_bus):
         @crewai_event_bus.on(CrewKickoffStartedEvent)
         def on_crew_started(source, event: CrewKickoffStartedEvent):
-            # Assuming that this event is called in the crew.kickoff method
             current_span = current_span_context.get()
-
-            # set the input
             if current_span:
                 current_span.input = event.inputs
-
-            # set trace input
             current_trace = current_trace_context.get()
             if current_trace:
                 current_trace.input = event.inputs
 
         @crewai_event_bus.on(CrewKickoffCompletedEvent)
         def on_crew_completed(source, event: CrewKickoffCompletedEvent):
-            # Assuming that this event is called in the crew.kickoff method
             current_span = current_span_context.get()
-
-            # set the output
+            output = getattr(
+                event, "output", getattr(event, "result", str(event))
+            )
             if current_span:
-                current_span.output = str(event.output)
-
-            # set trace output
+                current_span.output = str(output)
             current_trace = current_trace_context.get()
             if current_trace:
-                current_trace.output = str(event.output)
+                current_trace.output = str(output)
 
         @crewai_event_bus.on(LLMCallStartedEvent)
         def on_llm_started(source, event: LLMCallStartedEvent):
-            # Assuming that this event is called in the llm.call method
-            current_span = current_span_context.get()
-
-            # set the input
-            if current_span:
-                current_span.input = event.messages
+            metric_collection, metrics = _get_metrics_data(source)
+            observer = Observer(
+                span_type="llm",
+                func_name="call",
+                observe_kwargs={"model": getattr(event, "model", "unknown")},
+                metric_collection=metric_collection,
+                metrics=metrics,
+            )
+            self.span_observers[self.get_llm_execution_id(source, event)] = (
+                observer
+            )
+            observer.__enter__()
 
-                # set the model
-                if isinstance(current_span, LlmSpan):
-                    current_span.model = event.model
+            if observer.trace_uuid:
+                span = trace_manager.get_span_by_uuid(observer.uuid)
+                if span:
+                    msgs = getattr(event, "messages")
+                    span.input = msgs
 
         @crewai_event_bus.on(LLMCallCompletedEvent)
         def on_llm_completed(source, event: LLMCallCompletedEvent):
-            # Assuming that this event is called in the llm.call method
-            current_span = current_span_context.get()
-
-            # set the output
-            if current_span:
-                current_span.output = event.response
+            key = self.get_llm_execution_id(source, event)
+            if key in self.span_observers:
+                observer = self.span_observers.pop(key)
+                if observer:
+                    current_span = current_span_context.get()
+                    token = None
+                    span_to_close = trace_manager.get_span_by_uuid(
+                        observer.uuid
+                    )
+
+                    if span_to_close:
+                        output = getattr(
+                            event, "response", getattr(event, "output", "")
+                        )
+                        span_to_close.output = output
+                        if (
+                            not current_span
+                            or current_span.uuid != observer.uuid
+                        ):
+                            token = current_span_context.set(span_to_close)
+
+                    observer.__exit__(None, None, None)
+                    if token:
+                        current_span_context.reset(token)
 
         @crewai_event_bus.on(AgentExecutionStartedEvent)
         def on_agent_started(source, event: AgentExecutionStartedEvent):
-            # Assuming that this event is called in the agent.execute_task method
             current_span = current_span_context.get()
-
-            # set the input
             if current_span:
                 current_span.input = event.task_prompt
 
         @crewai_event_bus.on(AgentExecutionCompletedEvent)
         def on_agent_completed(source, event: AgentExecutionCompletedEvent):
-            # Assuming that this event is called in the agent.execute_task method
             current_span = current_span_context.get()
-
-            # set the output
             if current_span:
-                current_span.output = event.output
+                current_span.output = getattr(
+                    event, "output", getattr(event, "result", "")
+                )
 
         @crewai_event_bus.on(ToolUsageStartedEvent)
         def on_tool_started(source, event: ToolUsageStartedEvent):
+            key = self.get_tool_stack_key(source, event.tool_name)
+
+            # 1. Internal Stack Check
+            if self.tool_observers_stack[key]:
+                self.tool_observers_stack[key].append(None)
+                return
+
+            # 2. SMART DEDUPING
+            current_span = current_span_context.get()
+            span_type = getattr(current_span, "type", None)
+            is_tool_span = span_type == "tool" or span_type == SpanType.TOOL
+            if (
+                is_tool_span
+                and getattr(current_span, "name", "") == event.tool_name
+            ):
+                self.tool_observers_stack[key].append(None)
+                return
+
+            metric_collection = None
+            metrics = None
+
+            if hasattr(source, "tools"):
+                for tools in source.tools:
+                    if getattr(tools, "name", None) == event.tool_name:
+                        metric_collection, metrics = _get_metrics_data(tools)
+                        break
+
+            if not metric_collection:
+                agent = getattr(source, "agent", source)
+                metric_collection, metrics = _get_metrics_data(agent)
+
             observer = Observer(
                 span_type="tool",
                 func_name=event.tool_name,
                 function_kwargs=event.tool_args,
+                metric_collection=metric_collection,
+                metrics=metrics,
             )
-            self.span_observers[self.get_tool_execution_id(source, event)] = (
-                observer
-            )
+
+            self.tool_observers_stack[key].append(observer)
             observer.__enter__()
 
         @crewai_event_bus.on(ToolUsageFinishedEvent)
         def on_tool_completed(source, event: ToolUsageFinishedEvent):
-            observer = self.span_observers.pop(
-                self.get_tool_execution_id(source, event)
-            )
+            key = self.get_tool_stack_key(source, event.tool_name)
+            observer = None
+
+            if (
+                key in self.tool_observers_stack
+                and self.tool_observers_stack[key]
+            ):
+                item = self.tool_observers_stack[key].pop()
+                if item is None:
+                    return
+                observer = item
+
+            if not observer:
+                current_span = current_span_context.get()
+                if (
+                    current_span
+                    and getattr(current_span, "type", None)
+                    in ["tool", SpanType.TOOL]
+                    and getattr(current_span, "name", "") == event.tool_name
+                ):
+                    current_span.output = getattr(
+                        event, "output", getattr(event, "result", None)
+                    )
+
+                    if current_span.end_time is None:
+                        current_span.end_time = perf_counter()
+
+                    current_span.status = TraceSpanStatus.SUCCESS
+
+                    self._flatten_tool_span(current_span)
+                    trace_manager.remove_span(current_span.uuid)
+
+                    if current_span.parent_uuid:
+                        parent = trace_manager.get_span_by_uuid(
+                            current_span.parent_uuid
+                        )
+                        current_span_context.set(parent if parent else None)
+                    else:
+                        current_span_context.set(None)
+                    return
+
             if observer:
                 current_span = current_span_context.get()
-                if current_span:
-                    current_span.output = event.output
+                token = None
+                span_to_close = trace_manager.get_span_by_uuid(observer.uuid)
+
+                if span_to_close:
+                    span_to_close.output = getattr(
+                        event, "output", getattr(event, "result", None)
+                    )
+                    if not current_span or current_span.uuid != observer.uuid:
+                        token = current_span_context.set(span_to_close)
+
+                observer.update_span_properties = self._flatten_tool_span
                 observer.__exit__(None, None, None)
 
+                if span_to_close and span_to_close.end_time is None:
+                    span_to_close.end_time = perf_counter()
+                    span_to_close.status = TraceSpanStatus.SUCCESS
+
+                if token:
+                    current_span_context.reset(token)
+
         @crewai_event_bus.on(KnowledgeRetrievalStartedEvent)
         def on_knowledge_started(source, event: KnowledgeRetrievalStartedEvent):
             observer = Observer(
@@ -187,18 +354,35 @@ class CrewAIEventsListener(BaseEventListener):
         def on_knowledge_completed(
             source, event: KnowledgeRetrievalCompletedEvent
         ):
-            observer = self.span_observers.pop(
-                self.get_knowledge_execution_id(source, event)
-            )
-            if observer:
-                current_span = current_span_context.get()
-                if current_span:
-                    current_span.input = event.query
-                    current_span.output = event.retrieved_knowledge
-                observer.__exit__(None, None, None)
+            key = self.get_knowledge_execution_id(source, event)
+            if key in self.span_observers:
+                observer = self.span_observers.pop(key)
+                if observer:
+                    current_span = current_span_context.get()
+                    token = None
+                    span_to_close = trace_manager.get_span_by_uuid(
+                        observer.uuid
+                    )
+
+                    if span_to_close:
+                        span_to_close.input = event.query
+                        span_to_close.output = event.retrieved_knowledge
+
+                        if (
+                            not current_span
+                            or current_span.uuid != observer.uuid
+                        ):
+                            token = current_span_context.set(span_to_close)
+
+                    observer.__exit__(None, None, None)
+
+                    if token:
+                        current_span_context.reset(token)
 
 
 def instrument_crewai(api_key: Optional[str] = None):
+    global _listener_instance
+
     is_crewai_installed()
     with capture_tracing_integration("crewai"):
         if api_key:
@@ -206,7 +390,14 @@ def instrument_crewai(api_key: Optional[str] = None):
 
         wrap_all()
 
-        CrewAIEventsListener()
+        if _listener_instance is None:
+            _listener_instance = CrewAIEventsListener()
+
+
+def reset_crewai_instrumentation():
+    global _listener_instance
+    if _listener_instance:
+        _listener_instance.reset_state()
 
 
 def wrap_all():
@@ -218,15 +409,19 @@ def wrap_all():
             wrap_crew_kickoff_for_each,
             wrap_crew_kickoff_async,
             wrap_crew_kickoff_for_each_async,
-            wrap_llm_call,
+            wrap_crew_akickoff,
+            wrap_crew_akickoff_for_each,
             wrap_agent_execute_task,
+            wrap_agent_aexecute_task,
         )
 
         wrap_crew_kickoff()
         wrap_crew_kickoff_for_each()
         wrap_crew_kickoff_async()
         wrap_crew_kickoff_for_each_async()
-        wrap_llm_call()
+        wrap_crew_akickoff()
+        wrap_crew_akickoff_for_each()
         wrap_agent_execute_task()
+        wrap_agent_aexecute_task()
 
         IS_WRAPPED_ALL = True

deepeval/integrations/crewai/subs.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Type, TypeVar
+from typing import List, Optional, Type, TypeVar, Callable
 from pydantic import PrivateAttr
 
 from deepeval.metrics.base_metric import BaseMetric
@@ -28,14 +28,10 @@ def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
         _metric_collection: Optional[str] = PrivateAttr(default=None)
         _metrics: Optional[List[BaseMetric]] = PrivateAttr(default=None)
 
-        def __init__(
-            self,
-            *args,
-            metrics: Optional[List[BaseMetric]] = None,
-            metric_collection: Optional[str] = None,
-            **kwargs
-        ):
+        def __init__(self, *args, **kwargs):
             is_crewai_installed()
+            metric_collection = kwargs.pop("metric_collection", None)
+            metrics = kwargs.pop("metrics", None)
             super().__init__(*args, **kwargs)
             self._metric_collection = metric_collection
             self._metrics = metrics
@@ -45,7 +41,24 @@ def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
     return DeepEvalClass
 
 
-# Create the classes
+def create_deepeval_llm(base_factory: Callable) -> Callable:
+    """Wrapper for factory functions/classes (LLM)."""
+
+    def factory_wrapper(*args, **kwargs):
+        is_crewai_installed()
+        metric_collection = kwargs.pop("metric_collection", None)
+        metrics = kwargs.pop("metrics", None)
+        instance = base_factory(*args, **kwargs)
+        try:
+            instance._metric_collection = metric_collection
+            instance._metrics = metrics
+        except Exception:
+            pass
+        return instance
+
+    return factory_wrapper
+
+
 DeepEvalCrew = create_deepeval_class(Crew, "DeepEvalCrew")
 DeepEvalAgent = create_deepeval_class(Agent, "DeepEvalAgent")
-DeepEvalLLM = create_deepeval_class(LLM, "DeepEvalLLM")
+DeepEvalLLM = create_deepeval_llm(LLM)

deepeval/integrations/crewai/tool.py

@@ -15,6 +15,20 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
     """
     crewai_kwargs = kwargs
 
+    def _attach_metadata(tool_instance):
+        try:
+            object.__setattr__(
+                tool_instance, "metric_collection", metric_collection
+            )
+            object.__setattr__(tool_instance, "metrics", metric)
+        except Exception:
+            try:
+                tool_instance._metric_collection = metric_collection
+                tool_instance._metrics = metric
+            except Exception:
+                pass
+        return tool_instance
+
     # Case 1: @tool (function passed directly)
     if len(args) == 1 and callable(args[0]):
         f = args[0]
@@ -29,7 +43,8 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
             result = f(*f_args, **f_kwargs)
             return result
 
-        return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+        tool_instance = crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+        return _attach_metadata(tool_instance)
 
     # Case 2: @tool("name")
     if len(args) == 1 and isinstance(args[0], str):
@@ -45,7 +60,8 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
                 result = f(*f_args, **f_kwargs)
                 return result
 
-            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            tool_instance = crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            return _attach_metadata(tool_instance)
 
         return _decorator
 
@@ -64,7 +80,8 @@ def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
                 result = f(*f_args, **f_kwargs)
                 return result
 
-            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            tool_instance = crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+            return _attach_metadata(tool_instance)
 
         return _decorator
 

deepeval/integrations/crewai/wrapper.py

@@ -17,8 +17,9 @@ def wrap_crew_kickoff():
             func_name="kickoff",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_kickoff(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -36,8 +37,9 @@ def wrap_crew_kickoff_for_each():
             func_name="kickoff_for_each",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_kickoff_for_each(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -55,8 +57,9 @@ def wrap_crew_kickoff_async():
             func_name="kickoff_async",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = await original_kickoff_async(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -74,33 +77,61 @@ def wrap_crew_kickoff_for_each_async():
             func_name="kickoff_for_each_async",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = await original_kickoff_for_each_async(
                 self, *args, **kwargs
             )
+            observer.result = str(result) if result else None
 
         return result
 
     Crew.kickoff_for_each_async = wrapper
 
 
-def wrap_llm_call():
-    original_llm_call = LLM.call
+def wrap_crew_akickoff():
+    if not hasattr(Crew, "akickoff"):
+        return
 
-    @wraps(original_llm_call)
-    def wrapper(self, *args, **kwargs):
+    original_akickoff = Crew.akickoff
+
+    @wraps(original_akickoff)
+    async def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="akickoff",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ) as observer:
+            result = await original_akickoff(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+
+        return result
+
+    Crew.akickoff = wrapper
+
+
+def wrap_crew_akickoff_for_each():
+    if not hasattr(Crew, "akickoff_for_each"):
+        return
+
+    original_akickoff_for_each = Crew.akickoff_for_each
+
+    @wraps(original_akickoff_for_each)
+    async def wrapper(self, *args, **kwargs):
         metric_collection, metrics = _check_metrics_and_metric_collection(self)
         with Observer(
-            span_type="llm",
-            func_name="call",
-            observe_kwargs={"model": "temp_model"},
+            span_type="crew",
+            func_name="akickoff_for_each",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
-            result = original_llm_call(self, *args, **kwargs)
+        ) as observer:
+            result = await original_akickoff_for_each(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+
         return result
 
-    LLM.call = wrapper
+    Crew.akickoff_for_each = wrapper
 
 
 def wrap_agent_execute_task():
@@ -114,13 +145,36 @@ def wrap_agent_execute_task():
             func_name="execute_task",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_execute_task(self, *args, **kwargs)
+            observer.result = str(result) if result else None
         return result
 
     Agent.execute_task = wrapper
 
 
+def wrap_agent_aexecute_task():
+    if not hasattr(Agent, "aexecute_task"):
+        return
+
+    original_aexecute_task = Agent.aexecute_task
+
+    @wraps(original_aexecute_task)
+    async def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="agent",
+            func_name="aexecute_task",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ) as observer:
+            result = await original_aexecute_task(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+        return result
+
+    Agent.aexecute_task = wrapper
+
+
 def _check_metrics_and_metric_collection(obj: Any):
     metric_collection = getattr(obj, "_metric_collection", None)
     metrics = getattr(obj, "_metrics", None)

deepeval/integrations/langchain/callback.py

@@ -84,6 +84,7 @@ class CallbackHandler(BaseCallbackHandler):
         user_id: Optional[str] = None,
         metrics: Optional[List[BaseMetric]] = None,
         metric_collection: Optional[str] = None,
+        test_case_id: Optional[str] = None,
     ):
         is_langchain_installed()
         with capture_tracing_integration("langchain.callback.CallbackHandler"):
@@ -108,6 +109,7 @@ class CallbackHandler(BaseCallbackHandler):
                 "metadata": metadata,
                 "thread_id": thread_id,
                 "user_id": user_id,
+                "test_case_id": test_case_id,
             }
             self._trace_init_fields: Dict[str, Any] = dict(
                 self._original_init_fields
@@ -200,6 +202,8 @@ class CallbackHandler(BaseCallbackHandler):
                 trace.thread_id = fields["thread_id"]
             if fields.get("user_id") is not None:
                 trace.user_id = fields["user_id"]
+            if fields.get("test_case_id") is not None:
+                trace.test_case_id = fields["test_case_id"]
             # prevent re-applying on every callback
             self._trace_init_fields = {}
 

deepeval/integrations/llama_index/handler.py

@@ -21,6 +21,7 @@ from deepeval.tracing.types import (
 from deepeval.tracing.trace_context import (
     current_llm_context,
     current_agent_context,
+    current_trace_context,
 )
 from deepeval.test_case import ToolCall
 from deepeval.tracing.utils import make_json_serializable
@@ -40,7 +41,10 @@ try:
         LLMChatStartEvent,
         LLMChatEndEvent,
     )
-    from llama_index_instrumentation.dispatcher import Dispatcher
+    from llama_index.core.instrumentation import Dispatcher
+    from llama_index.core.instrumentation.events.retrieval import (
+        RetrievalEndEvent,
+    )
     from deepeval.integrations.llama_index.utils import (
         parse_id,
         prepare_input_llm_test_case_params,
@@ -82,15 +86,23 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 input_messages.append({"role": role, "content": content})
 
             llm_span_context = current_llm_context.get()
-            # create the span
+
+            parent_span = trace_manager.get_span_by_uuid(event.span_id)
+            if parent_span:
+                trace_uuid = parent_span.trace_uuid
+            else:
+                current_trace = current_trace_context.get()
+                if current_trace:
+                    trace_uuid = current_trace.uuid
+                else:
+                    trace_uuid = trace_manager.start_new_trace().uuid
+
             llm_span = LlmSpan(
                 name="ConfidentLLMSpan",
                 uuid=str(uuid.uuid4()),
                 status=TraceSpanStatus.IN_PROGRESS,
                 children=[],
-                trace_uuid=trace_manager.get_span_by_uuid(
-                    event.span_id
-                ).trace_uuid,
+                trace_uuid=trace_uuid,
                 parent_uuid=event.span_id,
                 start_time=perf_counter(),
                 model=getattr(event, "model_dict", {}).get(
@@ -128,6 +140,13 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     trace_manager.remove_span(llm_span.uuid)
                     del self.open_ai_astream_to_llm_span_map[event.span_id]
 
+        if isinstance(event, RetrievalEndEvent):
+            span = trace_manager.get_span_by_uuid(event.span_id)
+            if span:
+                span.retrieval_context = [
+                    node.node.get_content() for node in event.nodes
+                ]
+
     def new_span(
         self,
         id_: str,
@@ -139,18 +158,30 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
     ) -> Optional[LlamaIndexBaseSpan]:
         class_name, method_name = parse_id(id_)
 
-        # check if it is a root span
-        if parent_span_id is None:
-            trace_uuid = trace_manager.start_new_trace().uuid
-        elif class_name == "Workflow" and method_name == "run":
-            trace_uuid = trace_manager.start_new_trace().uuid
-            parent_span_id = None  # since workflow is the root span, we need to set the parent span id to None
+        current_trace = current_trace_context.get()
+        trace_uuid = None
+
+        if parent_span_id is None or (
+            class_name == "Workflow" and method_name == "run"
+        ):
+            if current_trace:
+                trace_uuid = current_trace.uuid
+            else:
+                trace_uuid = trace_manager.start_new_trace().uuid
+
+            if class_name == "Workflow" and method_name == "run":
+                parent_span_id = None
+
         elif trace_manager.get_span_by_uuid(parent_span_id):
             trace_uuid = trace_manager.get_span_by_uuid(
                 parent_span_id
             ).trace_uuid
+
         else:
-            trace_uuid = trace_manager.start_new_trace().uuid
+            if current_trace:
+                trace_uuid = current_trace.uuid
+            else:
+                trace_uuid = trace_manager.start_new_trace().uuid
 
         self.root_span_trace_id_map[id_] = trace_uuid
 
@@ -195,7 +226,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     else None
                 ),
             )
-        elif method_name == "acall":
+        elif method_name in ["acall", "call_tool", "acall_tool"]:
             span = ToolSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -206,7 +237,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 input=bound_args.arguments,
                 name="Tool",
             )
-        # prepare input test case params for the span
+
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments
         )
@@ -215,6 +246,22 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         return span
 
+    def _get_output_value(self, result: Any) -> Any:
+        """Helper to ensure AgentChatResponse and similar objects are serialized as dicts."""
+        if hasattr(result, "response") and hasattr(result, "sources"):
+            if hasattr(result, "model_dump"):
+                return result.model_dump()
+            if hasattr(result, "to_dict"):
+                return result.to_dict()
+            return {"response": result.response, "sources": result.sources}
+
+        if hasattr(result, "response"):
+            if hasattr(result, "model_dump"):
+                return result.model_dump()
+            return {"response": result.response}
+
+        return result
+
     def prepare_to_exit_span(
         self,
         id_: str,
@@ -229,7 +276,8 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
             return None
 
         class_name, method_name = parse_id(id_)
-        if method_name == "call_tool":
+
+        if method_name in ["call_tool", "acall_tool"]:
             output_json = make_json_serializable(result)
             if output_json and isinstance(output_json, dict):
                 if base_span.tools_called is None:
@@ -243,7 +291,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 )
         base_span.end_time = perf_counter()
         base_span.status = TraceSpanStatus.SUCCESS
-        base_span.output = result
+        base_span.output = self._get_output_value(result)
 
         if isinstance(base_span, ToolSpan):
             result_json = make_json_serializable(result)
@@ -265,7 +313,8 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         if base_span.parent_uuid is None:
             trace_manager.end_trace(base_span.trace_uuid)
-            self.root_span_trace_id_map.pop(base_span.uuid)
+            if base_span.uuid in self.root_span_trace_id_map:
+                self.root_span_trace_id_map.pop(base_span.uuid)
 
         return base_span
 
@@ -282,13 +331,12 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
             return None
 
         base_span.end_time = perf_counter()
-        base_span.status = (
-            TraceSpanStatus.SUCCESS
-        )  # find a way to add error and handle the span without the parent id
+        base_span.status = TraceSpanStatus.SUCCESS
 
         if base_span.parent_uuid is None:
             trace_manager.end_trace(base_span.trace_uuid)
-            self.root_span_trace_id_map.pop(base_span.uuid)
+            if base_span.uuid in self.root_span_trace_id_map:
+                self.root_span_trace_id_map.pop(base_span.uuid)
 
         return base_span
 

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -36,7 +36,10 @@ try:
         SpanProcessor as _SpanProcessor,
         TracerProvider,
     )
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace.export import (
+        BatchSpanProcessor,
+        SimpleSpanProcessor,
+    )
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
@@ -172,7 +175,9 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         trace_provider.add_span_processor(span_interceptor)
 
         if is_test_mode:
-            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+            trace_provider.add_span_processor(
+                SimpleSpanProcessor(ConfidentSpanExporter())
+            )
         else:
             trace_provider.add_span_processor(
                 BatchSpanProcessor(
@@ -345,7 +350,6 @@ class SpanInterceptor(SpanProcessor):
                 trace.status = TraceSpanStatus.SUCCESS
                 trace.end_time = perf_counter()
                 trace_manager.traces_to_evaluate.append(trace)
-                test_exporter.clear_span_json_list()
 
     def _add_agent_span(self, span, name):
         span.set_attribute("confident.span.type", "agent")

deepeval/tracing/api.py

@@ -126,6 +126,7 @@ class TraceApi(BaseModel):
     input: Optional[Any] = Field(None)
     output: Optional[Any] = Field(None)
     status: Optional[TraceSpanApiStatus] = Field(TraceSpanApiStatus.SUCCESS)
+    test_case_id: Optional[str] = Field(None, alias="testCaseId")
 
     # additional test case parameters
     retrieval_context: Optional[List[str]] = Field(

deepeval/tracing/context.py

@@ -74,6 +74,7 @@ def update_current_trace(
     expected_tools: Optional[List[ToolCall]] = None,
     test_case: Optional[LLMTestCase] = None,
     confident_api_key: Optional[str] = None,
+    test_case_id: Optional[str] = None,
 ):
     current_trace = current_trace_context.get()
     if not current_trace:
@@ -112,6 +113,8 @@ def update_current_trace(
         current_trace.expected_tools = expected_tools
     if confident_api_key:
         current_trace.confident_api_key = confident_api_key
+    if test_case_id:
+        current_trace.test_case_id = test_case_id
 
 
 def update_llm_span(

deepeval/tracing/trace_context.py

@@ -69,9 +69,11 @@ def trace(
         )
 
     current_trace = current_trace_context.get()
+    started_new_trace = False
 
     if not current_trace:
         current_trace = trace_manager.start_new_trace()
+        started_new_trace = True
 
     if metrics:
         current_trace.metrics = metrics
@@ -103,5 +105,8 @@ def trace(
     try:
         yield current_trace
     finally:
+        if started_new_trace:
+            trace_manager.end_trace(current_trace.uuid)
+
         current_llm_context.set(LlmSpanContext())
         current_agent_context.set(AgentSpanContext())

deepeval/tracing/tracing.py

@@ -690,6 +690,7 @@ class TraceManager:
             expectedOutput=trace.expected_output,
             toolsCalled=trace.tools_called,
             expectedTools=trace.expected_tools,
+            testCaseId=trace.test_case_id,
             confident_api_key=trace.confident_api_key,
             environment=(
                 self.environment if not trace.environment else trace.environment

deepeval/tracing/types.py

@@ -157,6 +157,7 @@ class Trace(BaseModel):
     output: Optional[Any] = None
     metrics: Optional[List[BaseMetric]] = None
     metric_collection: Optional[str] = None
+    test_case_id: Optional[str] = Field(None, serialization_alias="testCaseId")
 
     # Don't serialize these
     confident_api_key: Optional[str] = Field(None, exclude=True)

{deepeval-3.8.2.dist-info → deepeval-3.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.8.2
+Version: 3.8.3
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

{deepeval-3.8.2.dist-info → deepeval-3.8.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deepeval/__init__.py,sha256=tle4lT4FONApg3OeztGPEdrpGMEGLWajyGTu7bEd3s0,2976
-deepeval/_version.py,sha256=A8HjzlffHmJot3rBAExqN_D-QxaG8UT8zqiP26xCL2M,27
+deepeval/_version.py,sha256=DuZF4w5M7-EVzWNXFxp8yJIQADpdXSQSjeWW87pqtn4,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=WLFZRkx6wRJcNzaOMMGXuTfw6Q1_1Mv5A4jpD7Ea4sU,2300
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -169,26 +169,26 @@ deepeval/evaluate/execute.py,sha256=5RZrTRfe-AnwO5aS16LL-iBqT3fciun9zt3wbXp70v8,
 deepeval/evaluate/types.py,sha256=jf424xPHgdJcvgG2l_wTMskJBOEe9tl55c3v3B-SLNU,1071
 deepeval/evaluate/utils.py,sha256=STYyJCvVkewU5iigKnAsUDcVtJuFU_Qefi-aoyv2elA,20740
 deepeval/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/integrations/crewai/__init__.py,sha256=Uu7-MNV6uXnDoXlZJzt69Y73thJXlMM4xfQdv3mrlHw,227
-deepeval/integrations/crewai/handler.py,sha256=SbwosamuRQXOSmKPKKV7MMQ9vn-sDFiufCIHw-xxYKk,7882
-deepeval/integrations/crewai/subs.py,sha256=UP6aY8Nrl_lHDwxG84aRMNjxzP-X2QWTtOu9DgXMabM,1477
-deepeval/integrations/crewai/tool.py,sha256=6Uw3XuJO0STDq3YAL6bp_xh5ZJcza0L4kkuotFYj79s,2558
-deepeval/integrations/crewai/wrapper.py,sha256=r4szLNUz1O7U9eMps7czMbHyIEEQ4lfPdBb417gYPfc,3726
+deepeval/integrations/crewai/__init__.py,sha256=8bkOWfzrqETEYWMB9nCKbqMd7nlU-TGvNH0CIrqtlps,316
+deepeval/integrations/crewai/handler.py,sha256=dgzfWwHw94ro28-h8RyYfNQ8cOmnXCwEoWOZatxcbVk,15178
+deepeval/integrations/crewai/subs.py,sha256=wcMJVBNf69dqFO8L1kMTvm4n1GsO2S-nopQn924JdU4,1974
+deepeval/integrations/crewai/tool.py,sha256=thdDXt-dA2B4LmUuQipAcOiR6H19KA7zzqjlsfTyn3s,3210
+deepeval/integrations/crewai/wrapper.py,sha256=w3NsXHe1M4BhjGcpMA7grUCdSAjlu2zBk9daLeTRluc,5615
 deepeval/integrations/hugging_face/__init__.py,sha256=MuHIf9im9Jypq4VkfLzhklxIrd7vSTGlT74iUNSPgvg,93
 deepeval/integrations/hugging_face/callback.py,sha256=15QQEzR34Cpdp5kUp5oVA6dEsShtiMNZ03akJWAh7lo,7911
 deepeval/integrations/hugging_face/rich_manager.py,sha256=WvFtPGpPmGeg2Ftsnojga6yvbBLiZv_tvNbnFcGb6og,3630
 deepeval/integrations/hugging_face/tests/test_callbacks.py,sha256=88Wyg-aDaXujj9jHeGdFF3ITSl2-y7eaJGWgSyvvDi8,4607
 deepeval/integrations/hugging_face/utils.py,sha256=HUKdQcTIb76Ct69AS737oPxmlVxk5fw2UbT2pLn-o8k,1817
 deepeval/integrations/langchain/__init__.py,sha256=G1Qey5WkKou2-PA34KwWgmayQ_TbvXqPyotTbzmD8tw,84
-deepeval/integrations/langchain/callback.py,sha256=uZrhmlzw2dcFunqQzNPfMo9vWkHCX7PbGbazKRhBbBY,32687
+deepeval/integrations/langchain/callback.py,sha256=-Ip1PU84YqKbf4j17qV925GHsW5mRJN9b-d7V4fOuP8,32892
 deepeval/integrations/langchain/patch.py,sha256=fCHfZXU9xX3IJ6SG8GEYzn3qrifyUkT0i_uUABTsmcs,1255
 deepeval/integrations/langchain/utils.py,sha256=mhv0anU5ZnbBsESMuCooT9FSNPkx2ObrVLlq7QNEZOI,13104
 deepeval/integrations/llama_index/__init__.py,sha256=Ujs9ZBJFkuCWUDBJOF88UbM1Y-S6QFQhxSo0oQnEWNw,90
-deepeval/integrations/llama_index/handler.py,sha256=uTvNXmAF4xBh8t9bBm5sBFX6ETp8SrkOZlFlE_GWdmM,10771
+deepeval/integrations/llama_index/handler.py,sha256=uVfMs9VC2vp5J_A8lxy1OmVtha31wvkJGzkp5GKhf-A,12367
 deepeval/integrations/llama_index/utils.py,sha256=onmmo1vpn6cpOY5EhfTc0Uui7X6l1M0HD3sq-KVAesg,3380
 deepeval/integrations/pydantic_ai/__init__.py,sha256=UIkXn_g6h9LTQXG1PaWu1eCFkCssIwG48WSvN46UWgU,202
 deepeval/integrations/pydantic_ai/agent.py,sha256=-NKvpTUw3AxRNhuxVFcx9mw5BWCujzOwsaC8u7K0ubc,1178
-deepeval/integrations/pydantic_ai/instrumentator.py,sha256=COqw4FJsUZacaP4Dfn1aaOXvUTvZOuhcqqQD-_sLD04,13047
+deepeval/integrations/pydantic_ai/instrumentator.py,sha256=PPoGeJzkuoCerrjc-s_Nv8hn9DC54Jp-uXgJU0OI1Ug,13081
 deepeval/integrations/pydantic_ai/otel.py,sha256=xWYnMT1HwcAmyWdoJa6C1sHwd5frP9_IcR8dj9sKsG0,2386
 deepeval/integrations/pydantic_ai/test_instrumentator.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/key_handler.py,sha256=lajMBgF2lCzbQpW4e6Y7cD9FOw0Qk5UOKS4_kIIHj6Y,9562
@@ -501,8 +501,8 @@ deepeval/test_run/hooks.py,sha256=Qnd06bk9RJN4WmFUzJrBAi3Xj261hzyzI2iRmG8wbKw,37
 deepeval/test_run/hyperparameters.py,sha256=4yJkNgwL2y6eyWDTmUV62f5RUlfOui4R22wsJ5uTbto,3874
 deepeval/test_run/test_run.py,sha256=csbj0KVsp1QDCFKEqthQzPSmxusjNQkoVfsWBnq2Z_s,41549
 deepeval/tracing/__init__.py,sha256=aSOk_ZgL-K7CZzcyiaIa5peAiaPViDST5GhpHA3Adc8,614
-deepeval/tracing/api.py,sha256=GbtpUDdGpchl6rPXtZT6IBKjAhwux6qOlKLdP3dRVHU,4996
-deepeval/tracing/context.py,sha256=v4uzd0N2H8mNntPwZvL49ya2kW9FvjqQqWXJFgE5d0c,5469
+deepeval/tracing/api.py,sha256=x6Ze5ruPDbuRsR8rS0524cvUkCQ7CxLoT0up1gMQWMk,5062
+deepeval/tracing/context.py,sha256=KKP0Wp7zpzTzISyDceI7fndpVnehawI8rBMyRYEwb9U,5580
 deepeval/tracing/offline_evals/__init__.py,sha256=bEniJAl7PmS9u2ksiOTfHtlCPJ9_CJV5R6umrUOX5MM,102
 deepeval/tracing/offline_evals/api.py,sha256=eBfqh2uWyeRkIeGhjrN1bTQzAEow-XPubs-42WEZ2QQ,510
 deepeval/tracing/offline_evals/span.py,sha256=pXqTVXs-WnjRVpCYYEbNe0zSM6Wz9GsKHsM5ZcWxrmM,1802
@@ -514,14 +514,14 @@ deepeval/tracing/otel/test_exporter.py,sha256=bezihPGWJpwUEF3ZghxqhhorocVFTO2b43
 deepeval/tracing/otel/utils.py,sha256=NVMN07JtxuvVPtdyTW7KFdTqQL3TpoCO-JdZeghQJBY,17859
 deepeval/tracing/patchers.py,sha256=Oi9wao3oDYhcviv7p0KoWBeS9ne7rHLa2gh9AR9EyiU,6882
 deepeval/tracing/perf_epoch_bridge.py,sha256=iyAPddB6Op7NpMtPHJ29lDm53Btz9yLaN6xSCfTRQm4,1825
-deepeval/tracing/trace_context.py,sha256=Z0n0Cu1A5g9dXiZnzTFO5TzeOYHKeNuO6v3_EU_Gi_c,3568
+deepeval/tracing/trace_context.py,sha256=RkXSlAWuMzuRrvepsM0PlGaWwX7QBstkWv_frYn_7CE,3718
 deepeval/tracing/trace_test_manager.py,sha256=wt4y7EWTRc4Bw938-UFFtXHkdFFOrnx6JaIk7J5Iulw,555
-deepeval/tracing/tracing.py,sha256=ge3XXJkxlmCk5KfrqOOjxXIuA1CIXFOKJxhRTmXRSVQ,46849
-deepeval/tracing/types.py,sha256=3QkF0toQ6f0fEDARYOUV6Iv9UJFbg14kSpn3dL1H5CE,6040
+deepeval/tracing/tracing.py,sha256=RS3mBV-63_vDyz-WxYPir34u0BF3mPnAJWee1aCc1sc,46892
+deepeval/tracing/types.py,sha256=PUXDC1JZDaAalPc3uUHywkt2GE2hZ-2ocGP0Fe4sB2E,6120
 deepeval/tracing/utils.py,sha256=mdvhYAxDNsdnusaEXJd-c-_O2Jn6S3xSuzRvLO1Jz4U,5684
 deepeval/utils.py,sha256=Wsu95g6t1wdttxWIESVwuUxbml7C-9ZTsV7qHCQI3Xg,27259
-deepeval-3.8.2.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.8.2.dist-info/METADATA,sha256=SUHVBa7pgBKF2XG3L3c_cItJWvuCdAQxzQSctzeAezQ,18752
-deepeval-3.8.2.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.8.2.dist-info/entry_points.txt,sha256=NoismUQfwLOojSGZmBrdcpwfaoFRAzUhBvZD3UwOKog,95
-deepeval-3.8.2.dist-info/RECORD,,
+deepeval-3.8.3.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.8.3.dist-info/METADATA,sha256=BXj2MMYGn2usVl9VX5fcXNOHK-gp2DEY5uUDlP4kObQ,18752
+deepeval-3.8.3.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.8.3.dist-info/entry_points.txt,sha256=NoismUQfwLOojSGZmBrdcpwfaoFRAzUhBvZD3UwOKog,95
+deepeval-3.8.3.dist-info/RECORD,,