deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/evaluate/types.py

@@ -10,6 +10,7 @@ from deepeval.test_case import MLLMImage
 class TestResult:
     """Returned from run_test"""
 
+    __test__ = False
     name: str
     success: bool
     metrics_data: Union[List[MetricData], None]

deepeval/evaluate/utils.py

@@ -5,8 +5,6 @@ import os
 import time
 
 from deepeval.utils import format_turn
-from deepeval.test_case.conversational_test_case import Turn
-from deepeval.test_run.api import TurnApi
 from deepeval.test_run.test_run import TestRunResultDisplay
 from deepeval.dataset import Golden
 from deepeval.metrics import (
@@ -28,7 +26,6 @@ from deepeval.evaluate.types import TestResult
 from deepeval.tracing.api import TraceApi, BaseApiSpan, TraceSpanApiStatus
 from deepeval.tracing.tracing import BaseSpan, Trace
 from deepeval.tracing.types import TraceSpanStatus
-from deepeval.constants import PYTEST_RUN_TEST_NAME
 from deepeval.tracing.utils import (
     perf_counter_to_datetime,
     to_zod_compatible_iso,
@@ -133,121 +130,6 @@ def create_test_result(
             )
 
 
-def create_api_turn(turn: Turn, index: int) -> TurnApi:
-    return TurnApi(
-        role=turn.role,
-        content=turn.content,
-        user_id=turn.user_id,
-        retrievalContext=turn.retrieval_context,
-        toolsCalled=turn.tools_called,
-        additionalMetadata=turn.additional_metadata,
-        order=index,
-    )
-
-
-def create_api_test_case(
-    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
-    trace: Optional[TraceApi] = None,
-    index: Optional[int] = None,
-) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
-    if isinstance(test_case, ConversationalTestCase):
-        order = (
-            test_case._dataset_rank
-            if test_case._dataset_rank is not None
-            else index
-        )
-        if test_case.name:
-            name = test_case.name
-        else:
-            name = os.getenv(
-                PYTEST_RUN_TEST_NAME, f"conversational_test_case_{order}"
-            )
-
-        api_test_case = ConversationalApiTestCase(
-            name=name,
-            success=True,
-            metricsData=[],
-            runDuration=0,
-            evaluationCost=None,
-            order=order,
-            scenario=test_case.scenario,
-            expectedOutcome=test_case.expected_outcome,
-            userDescription=test_case.user_description,
-            context=test_case.context,
-            tags=test_case.tags,
-            comments=test_case.comments,
-            additionalMetadata=test_case.additional_metadata,
-        )
-        api_test_case.turns = [
-            create_api_turn(
-                turn=turn,
-                index=index,
-            )
-            for index, turn in enumerate(test_case.turns)
-        ]
-
-        return api_test_case
-    else:
-        order = (
-            test_case._dataset_rank
-            if test_case._dataset_rank is not None
-            else index
-        )
-
-        success = True
-        if test_case.name is not None:
-            name = test_case.name
-        else:
-            name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
-        metrics_data = []
-
-        if isinstance(test_case, LLMTestCase):
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input=test_case.input,
-                actualOutput=test_case.actual_output,
-                expectedOutput=test_case.expected_output,
-                context=test_case.context,
-                retrievalContext=test_case.retrieval_context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                tags=test_case.tags,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-                trace=trace,
-            )
-        elif isinstance(test_case, MLLMTestCase):
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input="",
-                multimodalInput=test_case.input,
-                multimodalActualOutput=test_case.actual_output,
-                multimodalExpectedOutput=test_case.expected_output,
-                multimodalRetrievalContext=test_case.retrieval_context,
-                multimodalContext=test_case.context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-            )
-        # llm_test_case_lookup_map[instance_id] = api_test_case
-        return api_test_case
-
-
 def create_api_trace(trace: Trace, golden: Golden) -> TraceApi:
     return TraceApi(
         uuid=trace.uuid,
@@ -309,6 +191,26 @@ def validate_assert_test_inputs(
             "Both 'test_case' and 'metrics' must be provided together."
         )
 
+    if test_case and metrics:
+        if isinstance(test_case, LLMTestCase) and not all(
+            isinstance(metric, BaseMetric) for metric in metrics
+        ):
+            raise ValueError(
+                "All 'metrics' for an 'LLMTestCase' must be instances of 'BaseMetric' only."
+            )
+        if isinstance(test_case, ConversationalTestCase) and not all(
+            isinstance(metric, BaseConversationalMetric) for metric in metrics
+        ):
+            raise ValueError(
+                "All 'metrics' for an 'ConversationalTestCase' must be instances of 'BaseConversationalMetric' only."
+            )
+        if isinstance(test_case, MLLMTestCase) and not all(
+            isinstance(metric, BaseMultimodalMetric) for metric in metrics
+        ):
+            raise ValueError(
+                "All 'metrics' for an 'MLLMTestCase' must be instances of 'BaseMultimodalMetric' only."
+            )
+
     if not ((golden and observed_callback) or (test_case and metrics)):
         raise ValueError(
             "You must provide either ('golden' + 'observed_callback') or ('test_case' + 'metrics')."
@@ -577,6 +479,18 @@ def count_metrics_in_trace(trace: Trace) -> int:
     return sum(count_metrics_recursive(span) for span in trace.root_spans)
 
 
+def count_total_metrics_for_trace(trace: Trace) -> int:
+    """Span subtree metrics + trace-level metrics."""
+    return count_metrics_in_trace(trace=trace) + len(trace.metrics or [])
+
+
+def count_metrics_in_span_subtree(span: BaseSpan) -> int:
+    total = len(span.metrics or [])
+    for c in span.children or []:
+        total += count_metrics_in_span_subtree(c)
+    return total
+
+
 def extract_trace_test_results(trace_api: TraceApi) -> List[TestResult]:
     test_results: List[TestResult] = []
     # extract trace result
@@ -619,7 +533,7 @@ def extract_span_test_results(span_api: BaseApiSpan) -> List[TestResult]:
         test_results.append(
             TestResult(
                 name=span_api.name,
-                success=span_api.status == "SUCCESS",
+                success=span_api.status == TraceSpanApiStatus.SUCCESS,
                 metrics_data=span_api.metrics_data,
                 input=span_api.input,
                 actual_output=span_api.output,

deepeval/integrations/crewai/__init__.py

@@ -1,3 +1,9 @@
 from .handler import instrument_crewai
+from .subs import (
+    DeepEvalCrew as Crew,
+    DeepEvalAgent as Agent,
+    DeepEvalLLM as LLM,
+)
+from .tool import tool
 
-__all__ = ["instrument_crewai"]
+__all__ = ["instrument_crewai", "Crew", "Agent", "LLM", "tool"]

deepeval/integrations/crewai/handler.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 try:
-    from crewai.utilities.events.base_event_listener import BaseEventListener
+    from crewai.events import BaseEventListener
     from crewai.events import (
         CrewKickoffStartedEvent,
         CrewKickoffCompletedEvent,

deepeval/integrations/crewai/subs.py

@@ -0,0 +1,51 @@
+from typing import List, Optional, Type, TypeVar
+from pydantic import PrivateAttr
+
+from deepeval.metrics.base_metric import BaseMetric
+
+try:
+    from crewai import Crew, Agent, LLM
+
+    is_crewai_installed = True
+except ImportError:
+    is_crewai_installed = False
+
+
+def is_crewai_installed():
+    if not is_crewai_installed:
+        raise ImportError(
+            "CrewAI is not installed. Please install it with `pip install crewai`."
+        )
+
+
+T = TypeVar("T")
+
+
+def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
+    """Factory function to create DeepEval-enabled CrewAI classes"""
+
+    class DeepEvalClass(base_class):
+        _metric_collection: Optional[str] = PrivateAttr(default=None)
+        _metrics: Optional[List[BaseMetric]] = PrivateAttr(default=None)
+
+        def __init__(
+            self,
+            *args,
+            metrics: Optional[List[BaseMetric]] = None,
+            metric_collection: Optional[str] = None,
+            **kwargs
+        ):
+            is_crewai_installed()
+            super().__init__(*args, **kwargs)
+            self._metric_collection = metric_collection
+            self._metrics = metrics
+
+    DeepEvalClass.__name__ = class_name
+    DeepEvalClass.__qualname__ = class_name
+    return DeepEvalClass
+
+
+# Create the classes
+DeepEvalCrew = create_deepeval_class(Crew, "DeepEvalCrew")
+DeepEvalAgent = create_deepeval_class(Agent, "DeepEvalAgent")
+DeepEvalLLM = create_deepeval_class(LLM, "DeepEvalLLM")

deepeval/integrations/crewai/tool.py

@@ -0,0 +1,71 @@
+import functools
+from typing import Callable
+from crewai.tools import tool as crewai_tool
+
+from deepeval.tracing.context import current_span_context
+from deepeval.tracing.types import ToolSpan
+
+
+def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
+    """
+    Simple wrapper around crewai.tools.tool that:
+      - prints the original function's input and output
+      - accepts additional parameters: metric and metric_collection (unused, for compatibility)
+      - remains backward compatible with CrewAI's decorator usage patterns
+    """
+    crewai_kwargs = kwargs
+
+    # Case 1: @tool (function passed directly)
+    if len(args) == 1 and callable(args[0]):
+        f = args[0]
+        tool_name = f.__name__
+
+        @functools.wraps(f)
+        def wrapped(*f_args, **f_kwargs):
+            current_span = current_span_context.get()
+            if current_span and isinstance(current_span, ToolSpan):
+                current_span.metric_collection = metric_collection
+                current_span.metrics = metric
+            result = f(*f_args, **f_kwargs)
+            return result
+
+        return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+
+    # Case 2: @tool("name")
+    if len(args) == 1 and isinstance(args[0], str):
+        tool_name = args[0]
+
+        def _decorator(f: Callable) -> Callable:
+            @functools.wraps(f)
+            def wrapped(*f_args, **f_kwargs):
+                current_span = current_span_context.get()
+                if current_span and isinstance(current_span, ToolSpan):
+                    current_span.metric_collection = metric_collection
+                    current_span.metrics = metric
+                result = f(*f_args, **f_kwargs)
+                return result
+
+            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+
+        return _decorator
+
+    # Case 3: @tool(result_as_answer=True, ...) — kwargs only
+    if len(args) == 0:
+
+        def _decorator(f: Callable) -> Callable:
+            tool_name = f.__name__
+
+            @functools.wraps(f)
+            def wrapped(*f_args, **f_kwargs):
+                current_span = current_span_context.get()
+                if current_span and isinstance(current_span, ToolSpan):
+                    current_span.metric_collection = metric_collection
+                    current_span.metrics = metric
+                result = f(*f_args, **f_kwargs)
+                return result
+
+            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+
+        return _decorator
+
+    raise ValueError("Invalid arguments")

deepeval/integrations/crewai/wrapper.py

@@ -3,6 +3,7 @@ from crewai.crew import Crew
 from crewai.agent import Agent
 from functools import wraps
 from deepeval.tracing.tracing import Observer
+from typing import Any
 
 
 def wrap_crew_kickoff():
@@ -10,7 +11,13 @@ def wrap_crew_kickoff():
 
     @wraps(original_kickoff)
     def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = original_kickoff(self, *args, **kwargs)
 
         return result
@@ -23,7 +30,13 @@ def wrap_crew_kickoff_for_each():
 
     @wraps(original_kickoff_for_each)
     def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff_for_each"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff_for_each",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = original_kickoff_for_each(self, *args, **kwargs)
 
         return result
@@ -36,7 +49,13 @@ def wrap_crew_kickoff_async():
 
     @wraps(original_kickoff_async)
     async def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff_async"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff_async",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = await original_kickoff_async(self, *args, **kwargs)
 
         return result
@@ -49,7 +68,13 @@ def wrap_crew_kickoff_for_each_async():
 
     @wraps(original_kickoff_for_each_async)
     async def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff_for_each_async"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff_for_each_async",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = await original_kickoff_for_each_async(
                 self, *args, **kwargs
             )
@@ -64,10 +89,13 @@ def wrap_llm_call():
 
     @wraps(original_llm_call)
     def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
         with Observer(
             span_type="llm",
             func_name="call",
             observe_kwargs={"model": "temp_model"},
+            metric_collection=metric_collection,
+            metrics=metrics,
         ):
             result = original_llm_call(self, *args, **kwargs)
         return result
@@ -80,8 +108,20 @@ def wrap_agent_execute_task():
 
     @wraps(original_execute_task)
     def wrapper(self, *args, **kwargs):
-        with Observer(span_type="agent", func_name="execute_task"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="agent",
+            func_name="execute_task",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = original_execute_task(self, *args, **kwargs)
         return result
 
     Agent.execute_task = wrapper
+
+
+def _check_metrics_and_metric_collection(obj: Any):
+    metric_collection = getattr(obj, "_metric_collection", None)
+    metrics = getattr(obj, "_metrics", None)
+    return metric_collection, metrics

deepeval/integrations/llama_index/__init__.py

@@ -1,10 +1,6 @@
 from .handler import instrument_llama_index
-from .agent.patched import FunctionAgent, ReActAgent, CodeActAgent
 
 
 __all__ = [
     "instrument_llama_index",
-    "FunctionAgent",
-    "ReActAgent",
-    "CodeActAgent",
 ]

deepeval/integrations/llama_index/handler.py

@@ -5,6 +5,10 @@ import uuid
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing import trace_manager
 from deepeval.tracing.types import AgentSpan, BaseSpan, LlmSpan, TraceSpanStatus
+from deepeval.tracing.trace_context import (
+    current_llm_context,
+    current_agent_context,
+)
 
 try:
     from llama_index.core.instrumentation.events.base import BaseEvent
@@ -22,11 +26,6 @@ try:
         LLMChatEndEvent,
     )
     from llama_index_instrumentation.dispatcher import Dispatcher
-    from deepeval.integrations.llama_index.agent.patched import (
-        FunctionAgent as PatchedFunctionAgent,
-        ReActAgent as PatchedReActAgent,
-        CodeActAgent as PatchedCodeActAgent,
-    )
     from deepeval.integrations.llama_index.utils import (
         parse_id,
         prepare_input_llm_test_case_params,
@@ -67,6 +66,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 ).strip()
                 input_messages.append({"role": role, "content": content})
 
+            llm_span_context = current_llm_context.get()
             # create the span
             llm_span = LlmSpan(
                 name="ConfidentLLMSpan",
@@ -83,6 +83,12 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 ),  # check the model name not coming in this option
                 input=input_messages,
                 output="",
+                metrics=llm_span_context.metrics if llm_span_context else None,
+                metric_collection=(
+                    llm_span_context.metric_collection
+                    if llm_span_context
+                    else None
+                ),
             )
             trace_manager.add_span(llm_span)
             trace_manager.add_span_to_trace(llm_span)
@@ -144,6 +150,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         # conditions to qualify as agent start run span
         if method_name == "run":
+            agent_span_context = current_agent_context.get()
             span = AgentSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -153,24 +160,16 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 start_time=perf_counter(),
                 name="Agent",  # TODO: decide the name of the span
                 input=bound_args.arguments,
+                metrics=(
+                    agent_span_context.metrics if agent_span_context else None
+                ),
+                metric_collection=(
+                    agent_span_context.metric_collection
+                    if agent_span_context
+                    else None
+                ),
             )
 
-            # check if the instance is a PatchedFunctionAgent
-            if isinstance(instance, PatchedFunctionAgent):
-                span.name = "FunctionAgent"
-                span.metric_collection = instance.metric_collection
-                span.metrics = instance.metrics
-
-            if isinstance(instance, PatchedReActAgent):
-                span.name = "ReActAgent"
-                span.metric_collection = instance.metric_collection
-                span.metrics = instance.metrics
-
-            if isinstance(instance, PatchedCodeActAgent):
-                span.name = "CodeActAgent"
-                span.metric_collection = instance.metric_collection
-                span.metrics = instance.metrics
-
         # prepare input test case params for the span
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments