deepeval 3.6.4__py3-none-any.whl → 3.6.6__py3-none-any.whl

deepeval/evaluate/utils.py

@@ -1,9 +1,10 @@
 import ast
 import inspect
-from typing import Optional, List, Callable, Union, Dict
-import os, time
-
+from typing import Optional, List, Callable, Union
+import os
+import time
 
+from deepeval.utils import format_turn
 from deepeval.test_case.conversational_test_case import Turn
 from deepeval.test_run.api import TurnApi
 from deepeval.test_run.test_run import TestRunResultDisplay
@@ -34,6 +35,29 @@ from deepeval.tracing.utils import (
 )
 
 
+def _is_metric_successful(metric_data: MetricData) -> bool:
+    """
+    Robustly determine success for a metric row.
+
+    Rationale:
+    - If the metric recorded an error, treat as failure.
+    - Be defensive: custom rows may not be MetricData at runtime.
+    """
+    if getattr(metric_data, "error", None):
+        return False
+
+    s = getattr(metric_data, "success", None)
+    if isinstance(s, bool):
+        return s
+    if s is None:
+        return False
+    if isinstance(s, (int, float)):
+        return bool(s)
+    if isinstance(s, str):
+        return s.strip().lower() in {"true", "t", "1", "yes", "y"}
+    return False
+
+
 def create_metric_data(metric: BaseMetric) -> MetricData:
     if metric.error is not None:
         return MetricData(
@@ -75,6 +99,7 @@ def create_test_result(
             metrics_data=api_test_case.metrics_data,
             conversational=True,
             additional_metadata=api_test_case.additional_metadata,
+            turns=api_test_case.turns,
         )
     else:
         multimodal = (
@@ -112,6 +137,7 @@ def create_api_turn(turn: Turn, index: int) -> TurnApi:
     return TurnApi(
         role=turn.role,
         content=turn.content,
+        user_id=turn.user_id,
         retrievalContext=turn.retrieval_context,
         toolsCalled=turn.tools_called,
         additionalMetadata=turn.additional_metadata,
@@ -372,17 +398,7 @@ def print_test_result(test_result: TestResult, display: TestRunResultDisplay):
     print("Metrics Summary\n")
 
     for metric_data in test_result.metrics_data:
-        successful = True
-        if metric_data.error is not None:
-            successful = False
-        else:
-            # This try block is for user defined custom metrics,
-            # which might not handle the score == undefined case elegantly
-            try:
-                if not metric_data.success:
-                    successful = False
-            except:
-                successful = False
+        successful = _is_metric_successful(metric_data)
 
         if not successful:
             print(
@@ -401,9 +417,14 @@ def print_test_result(test_result: TestResult, display: TestRunResultDisplay):
 
     elif test_result.conversational:
         print("For conversational test case:\n")
-        print(
-            f"  - Unable to print conversational test case. Run 'deepeval login' to view conversational evaluations in full."
-        )
+        if test_result.turns:
+            print("  Turns:")
+            turns = sorted(test_result.turns, key=lambda t: t.order)
+            for t in turns:
+                print(format_turn(t))
+        else:
+            print("  - No turns recorded in this test case.")
+
     else:
         print("For test case:\n")
         print(f"  - input: {test_result.input}")
@@ -470,15 +491,7 @@ def write_test_result_to_file(
         file.write("Metrics Summary\n\n")
 
         for metric_data in test_result.metrics_data:
-            successful = True
-            if metric_data.error is not None:
-                successful = False
-            else:
-                try:
-                    if not metric_data.success:
-                        successful = False
-                except:
-                    successful = False
+            successful = _is_metric_successful(metric_data)
 
             if not successful:
                 file.write(
@@ -500,9 +513,13 @@ def write_test_result_to_file(
             file.write(f"  - actual output: {test_result.actual_output}\n")
         elif test_result.conversational:
             file.write("For conversational test case:\n\n")
-            file.write(
-                "  - Unable to print conversational test case. Run 'deepeval login' to view conversational evaluations in full.\n"
-            )
+            if test_result.turns:
+                file.write("  Turns:\n")
+                turns = sorted(test_result.turns, key=lambda t: t.order)
+                for t in turns:
+                    file.write(format_turn(t) + "\n")
+            else:
+                file.write("  - No turns recorded in this test case.\n")
         else:
             file.write("For test case:\n\n")
             file.write(f"  - input: {test_result.input}\n")

deepeval/integrations/crewai/__init__.py

@@ -1,4 +1,3 @@
 from .handler import instrument_crewai
-from .agent import Agent
 
-__all__ = ["instrument_crewai", "Agent"]
+__all__ = ["instrument_crewai"]

deepeval/integrations/crewai/handler.py

@@ -1,30 +1,50 @@
-from typing import Optional
+import logging
 import deepeval
-from deepeval.integrations.crewai.agent import (
-    Agent as PatchedAgent,
-    agent_registry,
-)
-from deepeval.integrations.crewai.patch import patch_build_context_for_task
+
+from typing import Optional
 from deepeval.telemetry import capture_tracing_integration
-from deepeval.tracing.types import AgentSpan, LlmSpan
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.tracing.tracing import Observer
+from deepeval.tracing.types import LlmSpan
+from deepeval.config.settings import get_settings
+
+
+logger = logging.getLogger(__name__)
+
 
 try:
-    from crewai.crew import Crew
-    from crewai.llm import LLM
-    from crewai.agent import Agent
-    from crewai.utilities.events import AgentExecutionCompletedEvent
     from crewai.utilities.events.base_event_listener import BaseEventListener
-    from crewai.task import Task
-    from crewai.agents.crew_agent_executor import CrewAgentExecutor
-    from crewai.utilities.events import ToolUsageFinishedEvent
-    from crewai.tools.tool_usage import ToolUsage
-    from crewai.utilities.events import LLMCallCompletedEvent
-    from crewai.memory.contextual.contextual_memory import ContextualMemory
+    from crewai.events import (
+        CrewKickoffStartedEvent,
+        CrewKickoffCompletedEvent,
+        LLMCallStartedEvent,
+        LLMCallCompletedEvent,
+        AgentExecutionStartedEvent,
+        AgentExecutionCompletedEvent,
+        ToolUsageStartedEvent,
+        ToolUsageFinishedEvent,
+    )
 
     crewai_installed = True
-except:
+except ImportError as e:
+    if get_settings().DEEPEVAL_VERBOSE_MODE:
+        if isinstance(e, ModuleNotFoundError):
+            logger.warning(
+                "Optional crewai dependency not installed: %s",
+                e.name,
+                stacklevel=2,
+            )
+        else:
+            logger.warning(
+                "Optional crewai import failed: %s",
+                e,
+                stacklevel=2,
+            )
+
     crewai_installed = False
 
+IS_WRAPPED_ALL = False
+
 
 def is_crewai_installed():
     if not crewai_installed:
@@ -33,81 +53,114 @@ def is_crewai_installed():
         )
 
 
-from deepeval.test_case.llm_test_case import LLMTestCase
-from deepeval.tracing.tracing import (
-    observe,
-    current_span_context,
-    trace_manager,
-    current_trace_context,
-)
-
-
 class CrewAIEventsListener(BaseEventListener):
     def __init__(self):
         is_crewai_installed()
         super().__init__()
+        self.span_observers: dict[str, Observer] = {}
 
-    def setup_listeners(self, crewai_event_bus):
+    @staticmethod
+    def get_tool_execution_id(source, event) -> str:
+        source_id = id(source)
+        task_id = getattr(event, "task_id", "unknown")
+        agent_id = getattr(event, "agent_id", "unknown")
+        tool_name = getattr(event, "tool_name", "unknown")
+        execution_id = f"tool_{source_id}_{task_id}_{agent_id}_{tool_name}"
 
-        @crewai_event_bus.on(AgentExecutionCompletedEvent)
-        def on_agent_execution_completed(
-            source, event: AgentExecutionCompletedEvent
-        ):
+        return execution_id
+
+    def setup_listeners(self, crewai_event_bus):
+        @crewai_event_bus.on(CrewKickoffStartedEvent)
+        def on_crew_started(source, event: CrewKickoffStartedEvent):
+            # Assuming that this event is called in the crew.kickoff method
             current_span = current_span_context.get()
 
-            if isinstance(current_span, AgentSpan):
-                if isinstance(source, Agent):
-                    current_span.name = source.role
-                    current_span.available_tools = [
-                        tool.name for tool in source.tools
-                    ]
+            # set the input
+            if current_span:
+                current_span.input = event.inputs
+
+            # set trace input
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.input = event.inputs
+
+        @crewai_event_bus.on(CrewKickoffCompletedEvent)
+        def on_crew_completed(source, event: CrewKickoffCompletedEvent):
+            # Assuming that this event is called in the crew.kickoff method
+            current_span = current_span_context.get()
 
+            # set the output
             if current_span:
-                # set llm test case
-                input = None
-                actual_output = None
-                expected_output = None
-
-                if isinstance(event.task, Task):
-                    input = event.task.prompt()
-                    actual_output = event.output
-                    expected_output = event.task.expected_output
-
-                current_span.input = input
-                current_span.output = actual_output
-                current_span.expected_output = expected_output
-
-                # set metrics
-                if isinstance(source, PatchedAgent):
-                    current_span.metrics = agent_registry.get_metrics(source)
-                    current_span.metric_collection = (
-                        agent_registry.get_metric_collection(source)
-                    )
-
-                    # set offline evals
-                    if current_span.metric_collection:
-                        trace_manager.integration_traces_to_evaluate.append(
-                            current_trace_context.get()
-                        )
+                current_span.output = str(event.output)
 
-        @crewai_event_bus.on(ToolUsageFinishedEvent)
-        def on_tool_usage_finished(source, event: ToolUsageFinishedEvent):
+            # set trace output
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.output = str(event.output)
+
+        @crewai_event_bus.on(LLMCallStartedEvent)
+        def on_llm_started(source, event: LLMCallStartedEvent):
+            # Assuming that this event is called in the llm.call method
             current_span = current_span_context.get()
-            current_span.input = event.tool_args
-            current_span.output = event.output
-            current_span.name = event.tool_name
+
+            # set the input
+            if current_span:
+                current_span.input = event.messages
+
+                # set the model
+                if isinstance(current_span, LlmSpan):
+                    current_span.model = event.model
 
         @crewai_event_bus.on(LLMCallCompletedEvent)
-        def on_llm_call_finished(source, event: LLMCallCompletedEvent):
+        def on_llm_completed(source, event: LLMCallCompletedEvent):
+            # Assuming that this event is called in the llm.call method
             current_span = current_span_context.get()
 
-            if isinstance(current_span, LlmSpan):
-                if isinstance(source, LLM):
-                    current_span.model = source.model
-
-                current_span.input = event.messages
+            # set the output
+            if current_span:
                 current_span.output = event.response
 
+        @crewai_event_bus.on(AgentExecutionStartedEvent)
+        def on_agent_started(source, event: AgentExecutionStartedEvent):
+            # Assuming that this event is called in the agent.execute_task method
+            current_span = current_span_context.get()
+
+            # set the input
+            if current_span:
+                current_span.input = event.task_prompt
+
+        @crewai_event_bus.on(AgentExecutionCompletedEvent)
+        def on_agent_completed(source, event: AgentExecutionCompletedEvent):
+            # Assuming that this event is called in the agent.execute_task method
+            current_span = current_span_context.get()
+
+            # set the output
+            if current_span:
+                current_span.output = event.output
+
+        @crewai_event_bus.on(ToolUsageStartedEvent)
+        def on_tool_started(source, event: ToolUsageStartedEvent):
+            observer = Observer(
+                span_type="tool",
+                func_name=event.tool_name,
+                function_kwargs=event.tool_args,
+            )
+            self.span_observers[self.get_tool_execution_id(source, event)] = (
+                observer
+            )
+            observer.__enter__()
+
+        @crewai_event_bus.on(ToolUsageFinishedEvent)
+        def on_tool_completed(source, event: ToolUsageFinishedEvent):
+            observer = self.span_observers.pop(
+                self.get_tool_execution_id(source, event)
+            )
+            if observer:
+                current_span = current_span_context.get()
+                if current_span:
+                    current_span.output = event.output
+                observer.__exit__(None, None, None)
+
 
 def instrument_crewai(api_key: Optional[str] = None):
     is_crewai_installed()
@@ -115,10 +168,29 @@ def instrument_crewai(api_key: Optional[str] = None):
         if api_key:
             deepeval.login(api_key)
 
-        Crew.kickoff = observe(Crew.kickoff)
-        LLM.call = observe(LLM.call, type="llm", model="")
-        Agent.execute_task = observe(Agent.execute_task, type="agent")
-        CrewAgentExecutor.invoke = observe(CrewAgentExecutor.invoke)
-        ToolUsage.use = observe(ToolUsage.use, type="tool")
-        patch_build_context_for_task()
+        wrap_all()
+
         CrewAIEventsListener()
+
+
+def wrap_all():
+    global IS_WRAPPED_ALL
+
+    if not IS_WRAPPED_ALL:
+        from deepeval.integrations.crewai.wrapper import (
+            wrap_crew_kickoff,
+            wrap_crew_kickoff_for_each,
+            wrap_crew_kickoff_async,
+            wrap_crew_kickoff_for_each_async,
+            wrap_llm_call,
+            wrap_agent_execute_task,
+        )
+
+        wrap_crew_kickoff()
+        wrap_crew_kickoff_for_each()
+        wrap_crew_kickoff_async()
+        wrap_crew_kickoff_for_each_async()
+        wrap_llm_call()
+        wrap_agent_execute_task()
+
+        IS_WRAPPED_ALL = True

deepeval/integrations/crewai/wrapper.py

@@ -0,0 +1,87 @@
+from crewai.llm import LLM
+from crewai.crew import Crew
+from crewai.agent import Agent
+from functools import wraps
+from deepeval.tracing.tracing import Observer
+
+
+def wrap_crew_kickoff():
+    original_kickoff = Crew.kickoff
+
+    @wraps(original_kickoff)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff"):
+            result = original_kickoff(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff = wrapper
+
+
+def wrap_crew_kickoff_for_each():
+    original_kickoff_for_each = Crew.kickoff_for_each
+
+    @wraps(original_kickoff_for_each)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_for_each"):
+            result = original_kickoff_for_each(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff_for_each = wrapper
+
+
+def wrap_crew_kickoff_async():
+    original_kickoff_async = Crew.kickoff_async
+
+    @wraps(original_kickoff_async)
+    async def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_async"):
+            result = await original_kickoff_async(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff_async = wrapper
+
+
+def wrap_crew_kickoff_for_each_async():
+    original_kickoff_for_each_async = Crew.kickoff_for_each_async
+
+    @wraps(original_kickoff_for_each_async)
+    async def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_for_each_async"):
+            result = await original_kickoff_for_each_async(
+                self, *args, **kwargs
+            )
+
+        return result
+
+    Crew.kickoff_for_each_async = wrapper
+
+
+def wrap_llm_call():
+    original_llm_call = LLM.call
+
+    @wraps(original_llm_call)
+    def wrapper(self, *args, **kwargs):
+        with Observer(
+            span_type="llm",
+            func_name="call",
+            observe_kwargs={"model": "temp_model"},
+        ):
+            result = original_llm_call(self, *args, **kwargs)
+        return result
+
+    LLM.call = wrapper
+
+
+def wrap_agent_execute_task():
+    original_execute_task = Agent.execute_task
+
+    @wraps(original_execute_task)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="agent", func_name="execute_task"):
+            result = original_execute_task(self, *args, **kwargs)
+        return result
+
+    Agent.execute_task = wrapper

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -1,7 +1,19 @@
 import json
+import logging
 import os
 from typing import Literal, Optional, List
 
+from deepeval.config.settings import get_settings
+from deepeval.confident.api import get_confident_api_key
+from deepeval.prompt import Prompt
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import Trace
+from deepeval.tracing.otel.utils import to_hex_string
+
+
+logger = logging.getLogger(__name__)
+
+
 try:
     from pydantic_ai.models.instrumented import InstrumentationSettings
     from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
@@ -11,7 +23,20 @@ try:
     )
 
     dependency_installed = True
-except:
+except ImportError as e:
+    if get_settings().DEEPEVAL_VERBOSE_MODE:
+        if isinstance(e, ModuleNotFoundError):
+            logger.warning(
+                "Optional tracing dependency not installed: %s",
+                e.name,
+                stacklevel=2,
+            )
+        else:
+            logger.warning(
+                "Optional tracing import failed: %s",
+                e,
+                stacklevel=2,
+            )
     dependency_installed = False
 
 
@@ -25,6 +50,10 @@ def is_dependency_installed():
 
 from deepeval.confident.api import get_confident_api_key
 from deepeval.prompt import Prompt
+from deepeval.tracing.otel.test_exporter import test_exporter
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import Trace
+from deepeval.tracing.otel.utils import to_hex_string
 
 # OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
 OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
@@ -37,6 +66,12 @@ class SpanInterceptor(SpanProcessor):
 
     def on_start(self, span, parent_context):
 
+        # set trace uuid
+        _current_trace_context = current_trace_context.get()
+        if _current_trace_context and isinstance(_current_trace_context, Trace):
+            _otel_trace_id = span.get_span_context().trace_id
+            _current_trace_context.uuid = to_hex_string(_otel_trace_id, 32)
+
         # set trace attributes
         if self.settings.thread_id:
             span.set_attribute(
@@ -148,8 +183,9 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         confident_prompt: Optional[Prompt] = None,
         llm_metric_collection: Optional[str] = None,
         agent_metric_collection: Optional[str] = None,
-        tool_metric_collection_map: dict = {},
+        tool_metric_collection_map: Optional[dict] = None,
         trace_metric_collection: Optional[str] = None,
+        is_test_mode: Optional[bool] = False,
     ):
         is_dependency_installed()
 
@@ -162,7 +198,7 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         ]:
             self.environment = _environment
 
-        self.tool_metric_collection_map = tool_metric_collection_map
+        self.tool_metric_collection_map = tool_metric_collection_map or {}
         self.name = name
         self.thread_id = thread_id
         self.user_id = user_id
@@ -185,12 +221,15 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         span_interceptor = SpanInterceptor(self)
         trace_provider.add_span_processor(span_interceptor)
 
-        trace_provider.add_span_processor(
-            BatchSpanProcessor(
-                OTLPSpanExporter(
-                    endpoint=OTLP_ENDPOINT,
-                    headers={"x-confident-api-key": api_key},
+        if is_test_mode:
+            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+        else:
+            trace_provider.add_span_processor(
+                BatchSpanProcessor(
+                    OTLPSpanExporter(
+                        endpoint=OTLP_ENDPOINT,
+                        headers={"x-confident-api-key": api_key},
+                    )
                 )
             )
-        )
         super().__init__(tracer_provider=trace_provider)

deepeval/metrics/faithfulness/faithfulness.py

@@ -41,6 +41,7 @@ class FaithfulnessMetric(BaseMetric):
         strict_mode: bool = False,
         verbose_mode: bool = False,
         truths_extraction_limit: Optional[int] = None,
+        penalize_ambiguous_claims: bool = False,
         evaluation_template: Type[FaithfulnessTemplate] = FaithfulnessTemplate,
     ):
         self.threshold = 1 if strict_mode else threshold
@@ -51,6 +52,7 @@ class FaithfulnessMetric(BaseMetric):
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
         self.evaluation_template = evaluation_template
+        self.penalize_ambiguous_claims = penalize_ambiguous_claims
 
         self.truths_extraction_limit = truths_extraction_limit
         if self.truths_extraction_limit is not None:
@@ -329,6 +331,12 @@ class FaithfulnessMetric(BaseMetric):
             if verdict.verdict.strip().lower() != "no":
                 faithfulness_count += 1
 
+            if (
+                self.penalize_ambiguous_claims
+                and verdict.verdict.strip().lower() == "idk"
+            ):
+                faithfulness_count -= 1
+
         score = faithfulness_count / number_of_verdicts
         return 0 if self.strict_mode and score < self.threshold else score