deepeval 3.8.1__py3-none-any.whl → 3.8.3__py3-none-any.whl

deepeval/integrations/crewai/wrapper.py

@@ -17,8 +17,9 @@ def wrap_crew_kickoff():
             func_name="kickoff",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_kickoff(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -36,8 +37,9 @@ def wrap_crew_kickoff_for_each():
             func_name="kickoff_for_each",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_kickoff_for_each(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -55,8 +57,9 @@ def wrap_crew_kickoff_async():
             func_name="kickoff_async",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = await original_kickoff_async(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -74,33 +77,61 @@ def wrap_crew_kickoff_for_each_async():
             func_name="kickoff_for_each_async",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = await original_kickoff_for_each_async(
                 self, *args, **kwargs
             )
+            observer.result = str(result) if result else None
 
         return result
 
     Crew.kickoff_for_each_async = wrapper
 
 
-def wrap_llm_call():
-    original_llm_call = LLM.call
+def wrap_crew_akickoff():
+    if not hasattr(Crew, "akickoff"):
+        return
 
-    @wraps(original_llm_call)
-    def wrapper(self, *args, **kwargs):
+    original_akickoff = Crew.akickoff
+
+    @wraps(original_akickoff)
+    async def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="akickoff",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ) as observer:
+            result = await original_akickoff(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+
+        return result
+
+    Crew.akickoff = wrapper
+
+
+def wrap_crew_akickoff_for_each():
+    if not hasattr(Crew, "akickoff_for_each"):
+        return
+
+    original_akickoff_for_each = Crew.akickoff_for_each
+
+    @wraps(original_akickoff_for_each)
+    async def wrapper(self, *args, **kwargs):
         metric_collection, metrics = _check_metrics_and_metric_collection(self)
         with Observer(
-            span_type="llm",
-            func_name="call",
-            observe_kwargs={"model": "temp_model"},
+            span_type="crew",
+            func_name="akickoff_for_each",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
-            result = original_llm_call(self, *args, **kwargs)
+        ) as observer:
+            result = await original_akickoff_for_each(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+
         return result
 
-    LLM.call = wrapper
+    Crew.akickoff_for_each = wrapper
 
 
 def wrap_agent_execute_task():
@@ -114,13 +145,36 @@ def wrap_agent_execute_task():
             func_name="execute_task",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_execute_task(self, *args, **kwargs)
+            observer.result = str(result) if result else None
         return result
 
     Agent.execute_task = wrapper
 
 
+def wrap_agent_aexecute_task():
+    if not hasattr(Agent, "aexecute_task"):
+        return
+
+    original_aexecute_task = Agent.aexecute_task
+
+    @wraps(original_aexecute_task)
+    async def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="agent",
+            func_name="aexecute_task",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ) as observer:
+            result = await original_aexecute_task(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+        return result
+
+    Agent.aexecute_task = wrapper
+
+
 def _check_metrics_and_metric_collection(obj: Any):
     metric_collection = getattr(obj, "_metric_collection", None)
     metrics = getattr(obj, "_metrics", None)

deepeval/integrations/langchain/callback.py

@@ -1,3 +1,7 @@
+import logging
+import os
+import threading
+
 from typing import Any, Optional, List, Dict
 from uuid import UUID
 from time import perf_counter
@@ -20,6 +24,19 @@ from deepeval.tracing.types import (
 )
 from deepeval.telemetry import capture_tracing_integration
 
+# Debug logging for LangChain callbacks (enable with DEEPEVAL_DEBUG_LANGCHAIN_CALLBACKS=1)
+_DEBUG_CALLBACKS = os.environ.get(
+    "DEEPEVAL_DEBUG_LANGCHAIN_CALLBACKS", ""
+).lower() in ("1", "true", "yes")
+
+_logger = logging.getLogger(__name__)
+
+
+def _debug_log(msg: str):
+    if _DEBUG_CALLBACKS:
+        _logger.debug(f"[LangChain Callback] {msg}")
+
+
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
     from langchain_core.outputs import LLMResult
@@ -29,6 +46,7 @@ try:
     # contains langchain imports
     from deepeval.integrations.langchain.utils import (
         parse_prompts_to_messages,
+        convert_chat_messages_to_input,
         extract_name,
         safe_extract_model_name,
         safe_extract_token_usage,
@@ -50,6 +68,12 @@ def is_langchain_installed():
 
 
 class CallbackHandler(BaseCallbackHandler):
+    # When users create multiple CallbackHandler instances for the same logical
+    # conversation (same thread_id), we want spans to land on the same trace.
+    # Otherwise, each handler lazily creates its own trace, and multi-turn flows
+    # become multiple single-turn traces.
+    _thread_id_to_trace_uuid: Dict[str, str] = {}
+    _thread_id_lock = threading.Lock()
 
     def __init__(
         self,
@@ -60,6 +84,7 @@ class CallbackHandler(BaseCallbackHandler):
         user_id: Optional[str] = None,
         metrics: Optional[List[BaseMetric]] = None,
         metric_collection: Optional[str] = None,
+        test_case_id: Optional[str] = None,
     ):
         is_langchain_installed()
         with capture_tracing_integration("langchain.callback.CallbackHandler"):
@@ -74,13 +99,21 @@ class CallbackHandler(BaseCallbackHandler):
             self._parent_span = None
 
             # Stash trace metadata to apply once we know which trace we are using.
-            self._trace_init_fields: Dict[str, Any] = {
+            # _trace_init_fields is cleared after first apply to prevent re-applying
+            # on every callback within the same trace. _original_init_fields is kept
+            # permanently so we can re-apply when a new trace is created (e.g., in
+            # multi-turn scenarios where the previous trace was ended).
+            self._original_init_fields: Dict[str, Any] = {
                 "name": name,
                 "tags": tags,
                 "metadata": metadata,
                 "thread_id": thread_id,
                 "user_id": user_id,
+                "test_case_id": test_case_id,
             }
+            self._trace_init_fields: Dict[str, Any] = dict(
+                self._original_init_fields
+            )
 
             # Map LangChain run_id -> our span uuid for parent span restoration
             self._run_id_to_span_uuid: Dict[str, str] = {}
@@ -96,6 +129,34 @@ class CallbackHandler(BaseCallbackHandler):
         This is done lazily during actual callback execution to avoid context
         corruption when the handler is constructed outside the async task/context.
         """
+        # If the user provided a thread_id, attempt to reuse an existing trace for it.
+        # This makes multi-turn tests that use multiple CallbackHandler instances behave
+        # as expected: one trace containing multiple turns/spans.
+        thread_id = None
+        fields = self._trace_init_fields or {}
+        if fields.get("thread_id"):
+            thread_id = fields["thread_id"]
+        # In case _trace_init_fields has already been cleared, fall back to trace metadata.
+        if thread_id is None and self._trace is not None:
+            thread_id = self._trace.thread_id
+
+        if thread_id:
+            with self._thread_id_lock:
+                existing_uuid = self._thread_id_to_trace_uuid.get(thread_id)
+            if existing_uuid:
+                existing_trace = trace_manager.get_trace_by_uuid(existing_uuid)
+                if (
+                    existing_trace
+                    and existing_trace.uuid in trace_manager.active_traces
+                ):
+                    current_trace_context.set(existing_trace)
+                    self._trace = existing_trace
+                    self.trace_uuid = existing_trace.uuid
+                    # Lazily capture the observe parent span if present.
+                    if self._parent_span is None:
+                        self._parent_span = current_span_context.get()
+                    return existing_trace
+
         # Prefer current context trace if it is active.
         ctx_trace = current_trace_context.get()
         if ctx_trace and ctx_trace.uuid in trace_manager.active_traces:
@@ -107,6 +168,10 @@ class CallbackHandler(BaseCallbackHandler):
                 current_trace_context.set(trace)
             else:
                 # Otherwise, create a fresh trace now (in the right context).
+                # Restore _trace_init_fields from the original init fields so that
+                # the new trace gets the same name/tags/metadata as intended.
+                if not self._trace_init_fields and self._original_init_fields:
+                    self._trace_init_fields = dict(self._original_init_fields)
                 trace = trace_manager.start_new_trace()
                 current_trace_context.set(trace)
                 self._trace = trace
@@ -114,8 +179,18 @@ class CallbackHandler(BaseCallbackHandler):
         # Keep a copy for quick access.
         self.trace_uuid = trace.uuid
 
+        # Register this trace as the canonical trace for this thread_id (if provided).
+        # This allows other CallbackHandler instances created for the same thread_id
+        # to reuse the same trace instead of creating parallel traces.
+        fields = self._trace_init_fields or {}
+        tid = fields.get("thread_id") or trace.thread_id
+        if tid:
+            with self._thread_id_lock:
+                # Only set if absent to preserve the "first trace wins" behavior.
+                self._thread_id_to_trace_uuid.setdefault(tid, trace.uuid)
+
         # Apply stashed metadata once.
-        fields = getattr(self, "_trace_init_fields", None) or {}
+        fields = self._trace_init_fields or {}
         if fields:
             if fields.get("name") is not None:
                 trace.name = fields["name"]
@@ -127,6 +202,8 @@ class CallbackHandler(BaseCallbackHandler):
                 trace.thread_id = fields["thread_id"]
             if fields.get("user_id") is not None:
                 trace.user_id = fields["user_id"]
+            if fields.get("test_case_id") is not None:
+                trace.test_case_id = fields["test_case_id"]
             # prevent re-applying on every callback
             self._trace_init_fields = {}
 
@@ -202,6 +279,9 @@ class CallbackHandler(BaseCallbackHandler):
         metadata: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_chain_start: run_id={run_id}, parent_run_id={parent_run_id}, name={extract_name(serialized, **kwargs)}"
+        )
         # Create spans for all chains to establish proper parent-child hierarchy
         # This is important for LangGraph where there are nested chains
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
@@ -232,6 +312,9 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_chain_end: run_id={run_id}, parent_run_id={parent_run_id}"
+        )
         uuid_str = str(run_id)
         base_span = trace_manager.get_span_by_uuid(uuid_str)
         if base_span:
@@ -246,6 +329,59 @@ class CallbackHandler(BaseCallbackHandler):
                         trace.output = output
                 exit_current_context(uuid_str=uuid_str)
 
+    def on_chat_model_start(
+        self,
+        serialized: dict[str, Any],
+        messages: list[list[Any]],  # list[list[BaseMessage]]
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Handle chat model start callback. In LangChain v1, chat models emit
+        on_chat_model_start instead of on_llm_start. The on_llm_end callback
+        is still used for both.
+        """
+        _debug_log(
+            f"on_chat_model_start: run_id={run_id}, parent_run_id={parent_run_id}, messages_len={len(messages)}"
+        )
+
+        # Guard against double-counting if both on_llm_start and on_chat_model_start fire
+        uuid_str = str(run_id)
+        existing_span = trace_manager.get_span_by_uuid(uuid_str)
+        if existing_span is not None:
+            _debug_log(
+                f"on_chat_model_start: span already exists for run_id={run_id}, skipping"
+            )
+            return
+
+        with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
+            # Convert messages to our internal format using the shared helper
+            input_messages = convert_chat_messages_to_input(messages, **kwargs)
+
+            # Safe extraction of model name (handle None metadata)
+            md = metadata or {}
+            model = safe_extract_model_name(md, **kwargs)
+
+            llm_span: LlmSpan = enter_current_context(
+                uuid_str=uuid_str,
+                span_type="llm",
+                func_name=extract_name(serialized, **kwargs),
+            )
+            # Register this run_id -> span mapping for child callbacks
+            self._run_id_to_span_uuid[str(run_id)] = uuid_str
+
+            llm_span.input = input_messages
+            llm_span.model = model
+
+            # Extract metrics and prompt from metadata if provided, but don't mutate original
+            llm_span.metrics = md.get("metrics")
+            llm_span.metric_collection = md.get("metric_collection")
+            llm_span.prompt = md.get("prompt")
+
     def on_llm_start(
         self,
         serialized: dict[str, Any],
@@ -257,10 +393,25 @@ class CallbackHandler(BaseCallbackHandler):
         metadata: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_llm_start: run_id={run_id}, parent_run_id={parent_run_id}, prompts_len={len(prompts)}"
+        )
+
+        # Guard against double-counting if both on_llm_start and on_chat_model_start fire
+        uuid_str = str(run_id)
+        existing_span = trace_manager.get_span_by_uuid(uuid_str)
+        if existing_span is not None:
+            _debug_log(
+                f"on_llm_start: span already exists for run_id={run_id}, skipping"
+            )
+            return
+
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
-            uuid_str = str(run_id)
             input_messages = parse_prompts_to_messages(prompts, **kwargs)
-            model = safe_extract_model_name(metadata, **kwargs)
+
+            # Safe extraction of model name (handle None metadata)
+            md = metadata or {}
+            model = safe_extract_model_name(md, **kwargs)
 
             llm_span: LlmSpan = enter_current_context(
                 uuid_str=uuid_str,
@@ -272,12 +423,11 @@ class CallbackHandler(BaseCallbackHandler):
 
             llm_span.input = input_messages
             llm_span.model = model
-            metrics = metadata.pop("metrics", None)
-            metric_collection = metadata.pop("metric_collection", None)
-            prompt = metadata.pop("prompt", None)
-            llm_span.metrics = metrics
-            llm_span.metric_collection = metric_collection
-            llm_span.prompt = prompt
+
+            # Extract metrics and prompt from metadata if provided, but don't mutate original
+            llm_span.metrics = md.get("metrics")
+            llm_span.metric_collection = md.get("metric_collection")
+            llm_span.prompt = md.get("prompt")
 
     def on_llm_end(
         self,
@@ -287,9 +437,20 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
+        _debug_log(
+            f"on_llm_end: run_id={run_id}, parent_run_id={parent_run_id}, response_type={type(response).__name__}"
+        )
         uuid_str = str(run_id)
         llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
         if llm_span is None:
+            _debug_log(f"on_llm_end: NO SPAN FOUND for run_id={run_id}")
+            return
+
+        # Guard against double-finalization (if both on_llm_end and on_chat_model_end fire)
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_llm_end: span already finalized for run_id={run_id}, skipping"
+            )
             return
 
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
@@ -336,7 +497,6 @@ class CallbackHandler(BaseCallbackHandler):
                             )
 
             llm_span.model = model if model else llm_span.model
-            llm_span.input = llm_span.input
             llm_span.output = output
             llm_span.input_token_count = (
                 total_input_tokens if total_input_tokens > 0 else None
@@ -347,6 +507,121 @@ class CallbackHandler(BaseCallbackHandler):
 
             exit_current_context(uuid_str=uuid_str)
 
+    def on_chat_model_end(
+        self,
+        response: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Handle chat model end callback. This may be called instead of or
+        in addition to on_llm_end depending on the LangChain version.
+        """
+        _debug_log(
+            f"on_chat_model_end: run_id={run_id}, parent_run_id={parent_run_id}, response_type={type(response).__name__}"
+        )
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+        if llm_span is None:
+            _debug_log(f"on_chat_model_end: NO SPAN FOUND for run_id={run_id}")
+            return
+
+        # Guard against double-finalization, which could happen if both on_llm_end and on_chat_model_end fire
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_chat_model_end: span already finalized for run_id={run_id}, skipping"
+            )
+            return
+
+        with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
+            output = ""
+            total_input_tokens = 0
+            total_output_tokens = 0
+            model = None
+
+            # Handle LLMResult (same as on_llm_end)
+            if isinstance(response, LLMResult):
+                for generation in response.generations:
+                    for gen in generation:
+                        if isinstance(gen, ChatGeneration):
+                            if gen.message.response_metadata and isinstance(
+                                gen.message.response_metadata, dict
+                            ):
+                                model = gen.message.response_metadata.get(
+                                    "model_name"
+                                )
+                                input_tokens, output_tokens = (
+                                    safe_extract_token_usage(
+                                        gen.message.response_metadata
+                                    )
+                                )
+                                total_input_tokens += input_tokens
+                                total_output_tokens += output_tokens
+
+                            if isinstance(gen.message, AIMessage):
+                                ai_message = gen.message
+                                tool_calls = []
+                                for tool_call in ai_message.tool_calls:
+                                    tool_calls.append(
+                                        LlmToolCall(
+                                            name=tool_call["name"],
+                                            args=tool_call["args"],
+                                            id=tool_call["id"],
+                                        )
+                                    )
+                                output = LlmOutput(
+                                    role="AI",
+                                    content=ai_message.content,
+                                    tool_calls=tool_calls,
+                                )
+
+            llm_span.model = model if model else llm_span.model
+            llm_span.output = output
+            llm_span.input_token_count = (
+                total_input_tokens if total_input_tokens > 0 else None
+            )
+            llm_span.output_token_count = (
+                total_output_tokens if total_output_tokens > 0 else None
+            )
+
+            exit_current_context(uuid_str=uuid_str)
+
+    def on_chat_model_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Handle chat model error callback.
+        """
+        _debug_log(
+            f"on_chat_model_error: run_id={run_id}, parent_run_id={parent_run_id}, error={error}"
+        )
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+        if llm_span is None:
+            _debug_log(
+                f"on_chat_model_error: NO SPAN FOUND for run_id={run_id}"
+            )
+            return
+
+        # Guard against double-finalization
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_chat_model_error: span already finalized for run_id={run_id}, skipping"
+            )
+            return
+
+        with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
+            llm_span.status = TraceSpanStatus.ERRORED
+            llm_span.error = str(error)
+            exit_current_context(uuid_str=uuid_str)
+
     def on_llm_error(
         self,
         error: BaseException,
@@ -355,10 +630,22 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_llm_error: run_id={run_id}, parent_run_id={parent_run_id}, error={error}"
+        )
         uuid_str = str(run_id)
         llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
         if llm_span is None:
+            _debug_log(f"on_llm_error: NO SPAN FOUND for run_id={run_id}")
+            return
+
+        # Guard against double-finalization
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_llm_error: span already finalized for run_id={run_id}, skipping"
+            )
             return
+
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             llm_span.status = TraceSpanStatus.ERRORED
             llm_span.error = str(error)
@@ -396,6 +683,9 @@ class CallbackHandler(BaseCallbackHandler):
         inputs: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_tool_start: run_id={run_id}, parent_run_id={parent_run_id}, name={extract_name(serialized, **kwargs)}"
+        )
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             uuid_str = str(run_id)
 
@@ -418,6 +708,9 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
+        _debug_log(
+            f"on_tool_end: run_id={run_id}, parent_run_id={parent_run_id}"
+        )
         uuid_str = str(run_id)
         tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
         if tool_span is None:
@@ -485,20 +778,23 @@ class CallbackHandler(BaseCallbackHandler):
     ) -> Any:
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             uuid_str = str(run_id)
+            # Safe access to metadata (handle None)
+            md = metadata or {}
             retriever_span = enter_current_context(
                 uuid_str=uuid_str,
                 span_type="retriever",
                 func_name=extract_name(serialized, **kwargs),
                 observe_kwargs={
-                    "embedder": metadata.get(
-                        "ls_embedding_provider", "unknown"
-                    ),
+                    "embedder": md.get("ls_embedding_provider", "unknown"),
                 },
             )
             # Register this run_id -> span mapping for child callbacks
             self._run_id_to_span_uuid[str(run_id)] = uuid_str
             retriever_span.input = query
 
+            # Extract metric_collection from metadata if provided
+            retriever_span.metric_collection = md.get("metric_collection")
+
     def on_retriever_end(
         self,
         output: Any,