deepeval 3.6.3__py3-none-any.whl → 3.6.5__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.6.3"
+__version__: str = "3.6.5"

deepeval/config/settings.py

@@ -180,6 +180,19 @@ class Settings(BaseSettings):
     # into this directory. The directory will be created on demand.
     DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None
 
+    # Display / Truncation
+    DEEPEVAL_MAXLEN_TINY: Optional[int] = 40
+    DEEPEVAL_MAXLEN_SHORT: Optional[int] = 60
+    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = 120
+    DEEPEVAL_MAXLEN_LONG: Optional[int] = 240
+
+    # If set, this overrides the default max_len used by deepeval/utils shorten
+    # falls back to DEEPEVAL_MAXLEN_LONG when None.
+    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = None
+
+    # Optional global suffix (keeps your "..." default).
+    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = "..."
+
     #
     # GPU and perf toggles
     #

deepeval/dataset/dataset.py

@@ -1266,11 +1266,17 @@ class EvaluationDataset:
                 detach(ctx_token)
 
             else:
-                confident_link = global_test_run_manager.wrap_up_test_run(
+                res = global_test_run_manager.wrap_up_test_run(
                     run_duration, display_table=False
                 )
+                if isinstance(res, tuple):
+                    confident_link, test_run_id = res
+                else:
+                    confident_link = test_run_id = None
                 return EvaluationResult(
-                    test_results=test_results, confident_link=confident_link
+                    test_results=test_results,
+                    confident_link=confident_link,
+                    test_run_id=test_run_id,
                 )
 
     def evaluate(self, task: Task):

deepeval/evaluate/evaluate.py

@@ -268,11 +268,17 @@ def evaluate(
         test_run = global_test_run_manager.get_test_run()
         test_run.hyperparameters = process_hyperparameters(hyperparameters)
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
-        confident_link = global_test_run_manager.wrap_up_test_run(
+        res = global_test_run_manager.wrap_up_test_run(
             run_duration, display_table=False
         )
+        if isinstance(res, tuple):
+            confident_link, test_run_id = res
+        else:
+            confident_link = test_run_id = None
         return EvaluationResult(
-            test_results=test_results, confident_link=confident_link
+            test_results=test_results,
+            confident_link=confident_link,
+            test_run_id=test_run_id,
         )
     elif metric_collection:
         api = Api()

deepeval/evaluate/execute.py

@@ -45,9 +45,7 @@ from deepeval.dataset import Golden
 from deepeval.contextvars import set_current_golden, reset_current_golden
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.utils import copy_metrics
-from deepeval.utils import (
-    get_or_create_event_loop,
-)
+from deepeval.utils import get_or_create_event_loop, shorten, len_medium
 from deepeval.telemetry import capture_evaluation_run
 from deepeval.metrics import (
     BaseMetric,
@@ -1802,14 +1800,11 @@ def a_execute_agentic_test_cases_from_loop(
             )
 
             # record metadata for debugging
-            MAX_META_INPUT_LENGTH = 120
             started = time.perf_counter()
-            short_input = current_golden_ctx["input"]
-            if (
-                isinstance(short_input, str)
-                and len(short_input) > MAX_META_INPUT_LENGTH
-            ):
-                short_input = short_input[:MAX_META_INPUT_LENGTH] + "…"
+            short_input = current_golden_ctx.get("input")
+            if isinstance(short_input, str):
+                short_input = shorten(short_input, len_medium())
+
             task_meta[task] = {
                 "golden_index": current_golden_ctx["index"],
                 "golden_name": current_golden_ctx["name"],
@@ -1972,7 +1967,7 @@ def a_execute_agentic_test_cases_from_loop(
 
                 if settings.DEEPEVAL_DEBUG_ASYNC:
                     logger.warning(
-                        "[deepeval] %d stray task(s) not tracked; cancelling…",
+                        "[deepeval] %d stray task(s) not tracked; cancelling...",
                         len(leftovers),
                     )
                     for t in leftovers:

deepeval/evaluate/types.py

@@ -1,7 +1,8 @@
 from typing import Optional, List, Union, Dict
 from dataclasses import dataclass
 from pydantic import BaseModel
-from deepeval.test_run import MetricData
+
+from deepeval.test_run.api import MetricData, TurnApi
 from deepeval.test_case import MLLMImage
 
 
@@ -19,9 +20,11 @@ class TestResult:
     expected_output: Optional[str] = None
     context: Optional[List[str]] = None
     retrieval_context: Optional[List[str]] = None
+    turns: Optional[List[TurnApi]] = None
     additional_metadata: Optional[Dict] = None
 
 
 class EvaluationResult(BaseModel):
     test_results: List[TestResult]
     confident_link: Optional[str]
+    test_run_id: Optional[str]

deepeval/evaluate/utils.py

@@ -1,9 +1,10 @@
 import ast
 import inspect
-from typing import Optional, List, Callable, Union, Dict
-import os, time
-
+from typing import Optional, List, Callable, Union
+import os
+import time
 
+from deepeval.utils import format_turn
 from deepeval.test_case.conversational_test_case import Turn
 from deepeval.test_run.api import TurnApi
 from deepeval.test_run.test_run import TestRunResultDisplay
@@ -34,6 +35,29 @@ from deepeval.tracing.utils import (
 )
 
 
+def _is_metric_successful(metric_data: MetricData) -> bool:
+    """
+    Robustly determine success for a metric row.
+
+    Rationale:
+    - If the metric recorded an error, treat as failure.
+    - Be defensive: custom rows may not be MetricData at runtime.
+    """
+    if getattr(metric_data, "error", None):
+        return False
+
+    s = getattr(metric_data, "success", None)
+    if isinstance(s, bool):
+        return s
+    if s is None:
+        return False
+    if isinstance(s, (int, float)):
+        return bool(s)
+    if isinstance(s, str):
+        return s.strip().lower() in {"true", "t", "1", "yes", "y"}
+    return False
+
+
 def create_metric_data(metric: BaseMetric) -> MetricData:
     if metric.error is not None:
         return MetricData(
@@ -75,6 +99,7 @@ def create_test_result(
             metrics_data=api_test_case.metrics_data,
             conversational=True,
             additional_metadata=api_test_case.additional_metadata,
+            turns=api_test_case.turns,
         )
     else:
         multimodal = (
@@ -112,6 +137,7 @@ def create_api_turn(turn: Turn, index: int) -> TurnApi:
     return TurnApi(
         role=turn.role,
         content=turn.content,
+        user_id=turn.user_id,
         retrievalContext=turn.retrieval_context,
         toolsCalled=turn.tools_called,
         additionalMetadata=turn.additional_metadata,
@@ -372,17 +398,7 @@ def print_test_result(test_result: TestResult, display: TestRunResultDisplay):
     print("Metrics Summary\n")
 
     for metric_data in test_result.metrics_data:
-        successful = True
-        if metric_data.error is not None:
-            successful = False
-        else:
-            # This try block is for user defined custom metrics,
-            # which might not handle the score == undefined case elegantly
-            try:
-                if not metric_data.success:
-                    successful = False
-            except:
-                successful = False
+        successful = _is_metric_successful(metric_data)
 
         if not successful:
             print(
@@ -401,9 +417,14 @@ def print_test_result(test_result: TestResult, display: TestRunResultDisplay):
 
     elif test_result.conversational:
         print("For conversational test case:\n")
-        print(
-            f"  - Unable to print conversational test case. Run 'deepeval login' to view conversational evaluations in full."
-        )
+        if test_result.turns:
+            print("  Turns:")
+            turns = sorted(test_result.turns, key=lambda t: t.order)
+            for t in turns:
+                print(format_turn(t))
+        else:
+            print("  - No turns recorded in this test case.")
+
     else:
         print("For test case:\n")
         print(f"  - input: {test_result.input}")
@@ -470,15 +491,7 @@ def write_test_result_to_file(
         file.write("Metrics Summary\n\n")
 
         for metric_data in test_result.metrics_data:
-            successful = True
-            if metric_data.error is not None:
-                successful = False
-            else:
-                try:
-                    if not metric_data.success:
-                        successful = False
-                except:
-                    successful = False
+            successful = _is_metric_successful(metric_data)
 
             if not successful:
                 file.write(
@@ -500,9 +513,13 @@ def write_test_result_to_file(
             file.write(f"  - actual output: {test_result.actual_output}\n")
         elif test_result.conversational:
             file.write("For conversational test case:\n\n")
-            file.write(
-                "  - Unable to print conversational test case. Run 'deepeval login' to view conversational evaluations in full.\n"
-            )
+            if test_result.turns:
+                file.write("  Turns:\n")
+                turns = sorted(test_result.turns, key=lambda t: t.order)
+                for t in turns:
+                    file.write(format_turn(t) + "\n")
+            else:
+                file.write("  - No turns recorded in this test case.\n")
         else:
             file.write("For test case:\n\n")
             file.write(f"  - input: {test_result.input}\n")

deepeval/integrations/crewai/__init__.py

@@ -1,4 +1,3 @@
 from .handler import instrument_crewai
-from .agent import Agent
 
-__all__ = ["instrument_crewai", "Agent"]
+__all__ = ["instrument_crewai"]

deepeval/integrations/crewai/handler.py

@@ -1,30 +1,50 @@
-from typing import Optional
+import logging
 import deepeval
-from deepeval.integrations.crewai.agent import (
-    Agent as PatchedAgent,
-    agent_registry,
-)
-from deepeval.integrations.crewai.patch import patch_build_context_for_task
+
+from typing import Optional
 from deepeval.telemetry import capture_tracing_integration
-from deepeval.tracing.types import AgentSpan, LlmSpan
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.tracing.tracing import Observer
+from deepeval.tracing.types import LlmSpan
+from deepeval.config.settings import get_settings
+
+
+logger = logging.getLogger(__name__)
+
 
 try:
-    from crewai.crew import Crew
-    from crewai.llm import LLM
-    from crewai.agent import Agent
-    from crewai.utilities.events import AgentExecutionCompletedEvent
     from crewai.utilities.events.base_event_listener import BaseEventListener
-    from crewai.task import Task
-    from crewai.agents.crew_agent_executor import CrewAgentExecutor
-    from crewai.utilities.events import ToolUsageFinishedEvent
-    from crewai.tools.tool_usage import ToolUsage
-    from crewai.utilities.events import LLMCallCompletedEvent
-    from crewai.memory.contextual.contextual_memory import ContextualMemory
+    from crewai.events import (
+        CrewKickoffStartedEvent,
+        CrewKickoffCompletedEvent,
+        LLMCallStartedEvent,
+        LLMCallCompletedEvent,
+        AgentExecutionStartedEvent,
+        AgentExecutionCompletedEvent,
+        ToolUsageStartedEvent,
+        ToolUsageFinishedEvent,
+    )
 
     crewai_installed = True
-except:
+except ImportError as e:
+    if get_settings().DEEPEVAL_VERBOSE_MODE:
+        if isinstance(e, ModuleNotFoundError):
+            logger.warning(
+                "Optional crewai dependency not installed: %s",
+                e.name,
+                stacklevel=2,
+            )
+        else:
+            logger.warning(
+                "Optional crewai import failed: %s",
+                e,
+                stacklevel=2,
+            )
+
     crewai_installed = False
 
+IS_WRAPPED_ALL = False
+
 
 def is_crewai_installed():
     if not crewai_installed:
@@ -33,81 +53,114 @@ def is_crewai_installed():
         )
 
 
-from deepeval.test_case.llm_test_case import LLMTestCase
-from deepeval.tracing.tracing import (
-    observe,
-    current_span_context,
-    trace_manager,
-    current_trace_context,
-)
-
-
 class CrewAIEventsListener(BaseEventListener):
     def __init__(self):
         is_crewai_installed()
         super().__init__()
+        self.span_observers: dict[str, Observer] = {}
 
-    def setup_listeners(self, crewai_event_bus):
+    @staticmethod
+    def get_tool_execution_id(source, event) -> str:
+        source_id = id(source)
+        task_id = getattr(event, "task_id", "unknown")
+        agent_id = getattr(event, "agent_id", "unknown")
+        tool_name = getattr(event, "tool_name", "unknown")
+        execution_id = f"tool_{source_id}_{task_id}_{agent_id}_{tool_name}"
 
-        @crewai_event_bus.on(AgentExecutionCompletedEvent)
-        def on_agent_execution_completed(
-            source, event: AgentExecutionCompletedEvent
-        ):
+        return execution_id
+
+    def setup_listeners(self, crewai_event_bus):
+        @crewai_event_bus.on(CrewKickoffStartedEvent)
+        def on_crew_started(source, event: CrewKickoffStartedEvent):
+            # Assuming that this event is called in the crew.kickoff method
             current_span = current_span_context.get()
 
-            if isinstance(current_span, AgentSpan):
-                if isinstance(source, Agent):
-                    current_span.name = source.role
-                    current_span.available_tools = [
-                        tool.name for tool in source.tools
-                    ]
+            # set the input
+            if current_span:
+                current_span.input = event.inputs
+
+            # set trace input
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.input = event.inputs
+
+        @crewai_event_bus.on(CrewKickoffCompletedEvent)
+        def on_crew_completed(source, event: CrewKickoffCompletedEvent):
+            # Assuming that this event is called in the crew.kickoff method
+            current_span = current_span_context.get()
 
+            # set the output
             if current_span:
-                # set llm test case
-                input = None
-                actual_output = None
-                expected_output = None
-
-                if isinstance(event.task, Task):
-                    input = event.task.prompt()
-                    actual_output = event.output
-                    expected_output = event.task.expected_output
-
-                current_span.input = input
-                current_span.output = actual_output
-                current_span.expected_output = expected_output
-
-                # set metrics
-                if isinstance(source, PatchedAgent):
-                    current_span.metrics = agent_registry.get_metrics(source)
-                    current_span.metric_collection = (
-                        agent_registry.get_metric_collection(source)
-                    )
-
-                    # set offline evals
-                    if current_span.metric_collection:
-                        trace_manager.integration_traces_to_evaluate.append(
-                            current_trace_context.get()
-                        )
+                current_span.output = str(event.output)
 
-        @crewai_event_bus.on(ToolUsageFinishedEvent)
-        def on_tool_usage_finished(source, event: ToolUsageFinishedEvent):
+            # set trace output
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.output = str(event.output)
+
+        @crewai_event_bus.on(LLMCallStartedEvent)
+        def on_llm_started(source, event: LLMCallStartedEvent):
+            # Assuming that this event is called in the llm.call method
             current_span = current_span_context.get()
-            current_span.input = event.tool_args
-            current_span.output = event.output
-            current_span.name = event.tool_name
+
+            # set the input
+            if current_span:
+                current_span.input = event.messages
+
+                # set the model
+                if isinstance(current_span, LlmSpan):
+                    current_span.model = event.model
 
         @crewai_event_bus.on(LLMCallCompletedEvent)
-        def on_llm_call_finished(source, event: LLMCallCompletedEvent):
+        def on_llm_completed(source, event: LLMCallCompletedEvent):
+            # Assuming that this event is called in the llm.call method
             current_span = current_span_context.get()
 
-            if isinstance(current_span, LlmSpan):
-                if isinstance(source, LLM):
-                    current_span.model = source.model
-
-                current_span.input = event.messages
+            # set the output
+            if current_span:
                 current_span.output = event.response
 
+        @crewai_event_bus.on(AgentExecutionStartedEvent)
+        def on_agent_started(source, event: AgentExecutionStartedEvent):
+            # Assuming that this event is called in the agent.execute_task method
+            current_span = current_span_context.get()
+
+            # set the input
+            if current_span:
+                current_span.input = event.task_prompt
+
+        @crewai_event_bus.on(AgentExecutionCompletedEvent)
+        def on_agent_completed(source, event: AgentExecutionCompletedEvent):
+            # Assuming that this event is called in the agent.execute_task method
+            current_span = current_span_context.get()
+
+            # set the output
+            if current_span:
+                current_span.output = event.output
+
+        @crewai_event_bus.on(ToolUsageStartedEvent)
+        def on_tool_started(source, event: ToolUsageStartedEvent):
+            observer = Observer(
+                span_type="tool",
+                func_name=event.tool_name,
+                function_kwargs=event.tool_args,
+            )
+            self.span_observers[self.get_tool_execution_id(source, event)] = (
+                observer
+            )
+            observer.__enter__()
+
+        @crewai_event_bus.on(ToolUsageFinishedEvent)
+        def on_tool_completed(source, event: ToolUsageFinishedEvent):
+            observer = self.span_observers.pop(
+                self.get_tool_execution_id(source, event)
+            )
+            if observer:
+                current_span = current_span_context.get()
+                if current_span:
+                    current_span.output = event.output
+                observer.__exit__(None, None, None)
+
 
 def instrument_crewai(api_key: Optional[str] = None):
     is_crewai_installed()
@@ -115,10 +168,29 @@ def instrument_crewai(api_key: Optional[str] = None):
         if api_key:
             deepeval.login(api_key)
 
-        Crew.kickoff = observe(Crew.kickoff)
-        LLM.call = observe(LLM.call, type="llm", model="")
-        Agent.execute_task = observe(Agent.execute_task, type="agent")
-        CrewAgentExecutor.invoke = observe(CrewAgentExecutor.invoke)
-        ToolUsage.use = observe(ToolUsage.use, type="tool")
-        patch_build_context_for_task()
+        wrap_all()
+
         CrewAIEventsListener()
+
+
+def wrap_all():
+    global IS_WRAPPED_ALL
+
+    if not IS_WRAPPED_ALL:
+        from deepeval.integrations.crewai.wrapper import (
+            wrap_crew_kickoff,
+            wrap_crew_kickoff_for_each,
+            wrap_crew_kickoff_async,
+            wrap_crew_kickoff_for_each_async,
+            wrap_llm_call,
+            wrap_agent_execute_task,
+        )
+
+        wrap_crew_kickoff()
+        wrap_crew_kickoff_for_each()
+        wrap_crew_kickoff_async()
+        wrap_crew_kickoff_for_each_async()
+        wrap_llm_call()
+        wrap_agent_execute_task()
+
+        IS_WRAPPED_ALL = True

deepeval/integrations/crewai/wrapper.py

@@ -0,0 +1,87 @@
+from crewai.llm import LLM
+from crewai.crew import Crew
+from crewai.agent import Agent
+from functools import wraps
+from deepeval.tracing.tracing import Observer
+
+
+def wrap_crew_kickoff():
+    original_kickoff = Crew.kickoff
+
+    @wraps(original_kickoff)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff"):
+            result = original_kickoff(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff = wrapper
+
+
+def wrap_crew_kickoff_for_each():
+    original_kickoff_for_each = Crew.kickoff_for_each
+
+    @wraps(original_kickoff_for_each)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_for_each"):
+            result = original_kickoff_for_each(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff_for_each = wrapper
+
+
+def wrap_crew_kickoff_async():
+    original_kickoff_async = Crew.kickoff_async
+
+    @wraps(original_kickoff_async)
+    async def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_async"):
+            result = await original_kickoff_async(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff_async = wrapper
+
+
+def wrap_crew_kickoff_for_each_async():
+    original_kickoff_for_each_async = Crew.kickoff_for_each_async
+
+    @wraps(original_kickoff_for_each_async)
+    async def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_for_each_async"):
+            result = await original_kickoff_for_each_async(
+                self, *args, **kwargs
+            )
+
+        return result
+
+    Crew.kickoff_for_each_async = wrapper
+
+
+def wrap_llm_call():
+    original_llm_call = LLM.call
+
+    @wraps(original_llm_call)
+    def wrapper(self, *args, **kwargs):
+        with Observer(
+            span_type="llm",
+            func_name="call",
+            observe_kwargs={"model": "temp_model"},
+        ):
+            result = original_llm_call(self, *args, **kwargs)
+        return result
+
+    LLM.call = wrapper
+
+
+def wrap_agent_execute_task():
+    original_execute_task = Agent.execute_task
+
+    @wraps(original_execute_task)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="agent", func_name="execute_task"):
+            result = original_execute_task(self, *args, **kwargs)
+        return result
+
+    Agent.execute_task = wrapper