deepeval 3.6.3__tar.gz → 3.6.5__tar.gz

{deepeval-3.6.3 → deepeval-3.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.3
+Version: 3.6.5
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.6.5/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.6.5"

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/config/settings.py

@@ -180,6 +180,19 @@ class Settings(BaseSettings):
     # into this directory. The directory will be created on demand.
     DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None
 
+    # Display / Truncation
+    DEEPEVAL_MAXLEN_TINY: Optional[int] = 40
+    DEEPEVAL_MAXLEN_SHORT: Optional[int] = 60
+    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = 120
+    DEEPEVAL_MAXLEN_LONG: Optional[int] = 240
+
+    # If set, this overrides the default max_len used by deepeval/utils shorten
+    # falls back to DEEPEVAL_MAXLEN_LONG when None.
+    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = None
+
+    # Optional global suffix (keeps your "..." default).
+    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = "..."
+
     #
     # GPU and perf toggles
     #

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/dataset/dataset.py

@@ -1266,11 +1266,17 @@ class EvaluationDataset:
                 detach(ctx_token)
 
             else:
-                confident_link = global_test_run_manager.wrap_up_test_run(
+                res = global_test_run_manager.wrap_up_test_run(
                     run_duration, display_table=False
                 )
+                if isinstance(res, tuple):
+                    confident_link, test_run_id = res
+                else:
+                    confident_link = test_run_id = None
                 return EvaluationResult(
-                    test_results=test_results, confident_link=confident_link
+                    test_results=test_results,
+                    confident_link=confident_link,
+                    test_run_id=test_run_id,
                 )
 
     def evaluate(self, task: Task):

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/evaluate/evaluate.py

@@ -268,11 +268,17 @@ def evaluate(
         test_run = global_test_run_manager.get_test_run()
         test_run.hyperparameters = process_hyperparameters(hyperparameters)
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
-        confident_link = global_test_run_manager.wrap_up_test_run(
+        res = global_test_run_manager.wrap_up_test_run(
             run_duration, display_table=False
         )
+        if isinstance(res, tuple):
+            confident_link, test_run_id = res
+        else:
+            confident_link = test_run_id = None
         return EvaluationResult(
-            test_results=test_results, confident_link=confident_link
+            test_results=test_results,
+            confident_link=confident_link,
+            test_run_id=test_run_id,
         )
     elif metric_collection:
         api = Api()

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/evaluate/execute.py

@@ -45,9 +45,7 @@ from deepeval.dataset import Golden
 from deepeval.contextvars import set_current_golden, reset_current_golden
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.utils import copy_metrics
-from deepeval.utils import (
-    get_or_create_event_loop,
-)
+from deepeval.utils import get_or_create_event_loop, shorten, len_medium
 from deepeval.telemetry import capture_evaluation_run
 from deepeval.metrics import (
     BaseMetric,
@@ -1802,14 +1800,11 @@ def a_execute_agentic_test_cases_from_loop(
             )
 
             # record metadata for debugging
-            MAX_META_INPUT_LENGTH = 120
             started = time.perf_counter()
-            short_input = current_golden_ctx["input"]
-            if (
-                isinstance(short_input, str)
-                and len(short_input) > MAX_META_INPUT_LENGTH
-            ):
-                short_input = short_input[:MAX_META_INPUT_LENGTH] + "…"
+            short_input = current_golden_ctx.get("input")
+            if isinstance(short_input, str):
+                short_input = shorten(short_input, len_medium())
+
             task_meta[task] = {
                 "golden_index": current_golden_ctx["index"],
                 "golden_name": current_golden_ctx["name"],
@@ -1972,7 +1967,7 @@ def a_execute_agentic_test_cases_from_loop(
 
                 if settings.DEEPEVAL_DEBUG_ASYNC:
                     logger.warning(
-                        "[deepeval] %d stray task(s) not tracked; cancelling…",
+                        "[deepeval] %d stray task(s) not tracked; cancelling...",
                         len(leftovers),
                     )
                     for t in leftovers:

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/evaluate/types.py

@@ -1,7 +1,8 @@
 from typing import Optional, List, Union, Dict
 from dataclasses import dataclass
 from pydantic import BaseModel
-from deepeval.test_run import MetricData
+
+from deepeval.test_run.api import MetricData, TurnApi
 from deepeval.test_case import MLLMImage
 
 
@@ -19,9 +20,11 @@ class TestResult:
     expected_output: Optional[str] = None
     context: Optional[List[str]] = None
     retrieval_context: Optional[List[str]] = None
+    turns: Optional[List[TurnApi]] = None
     additional_metadata: Optional[Dict] = None
 
 
 class EvaluationResult(BaseModel):
     test_results: List[TestResult]
     confident_link: Optional[str]
+    test_run_id: Optional[str]

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/evaluate/utils.py

@@ -1,9 +1,10 @@
 import ast
 import inspect
-from typing import Optional, List, Callable, Union, Dict
-import os, time
-
+from typing import Optional, List, Callable, Union
+import os
+import time
 
+from deepeval.utils import format_turn
 from deepeval.test_case.conversational_test_case import Turn
 from deepeval.test_run.api import TurnApi
 from deepeval.test_run.test_run import TestRunResultDisplay
@@ -34,6 +35,29 @@ from deepeval.tracing.utils import (
 )
 
 
+def _is_metric_successful(metric_data: MetricData) -> bool:
+    """
+    Robustly determine success for a metric row.
+
+    Rationale:
+    - If the metric recorded an error, treat as failure.
+    - Be defensive: custom rows may not be MetricData at runtime.
+    """
+    if getattr(metric_data, "error", None):
+        return False
+
+    s = getattr(metric_data, "success", None)
+    if isinstance(s, bool):
+        return s
+    if s is None:
+        return False
+    if isinstance(s, (int, float)):
+        return bool(s)
+    if isinstance(s, str):
+        return s.strip().lower() in {"true", "t", "1", "yes", "y"}
+    return False
+
+
 def create_metric_data(metric: BaseMetric) -> MetricData:
     if metric.error is not None:
         return MetricData(
@@ -75,6 +99,7 @@ def create_test_result(
             metrics_data=api_test_case.metrics_data,
             conversational=True,
             additional_metadata=api_test_case.additional_metadata,
+            turns=api_test_case.turns,
         )
     else:
         multimodal = (
@@ -112,6 +137,7 @@ def create_api_turn(turn: Turn, index: int) -> TurnApi:
     return TurnApi(
         role=turn.role,
         content=turn.content,
+        user_id=turn.user_id,
         retrievalContext=turn.retrieval_context,
         toolsCalled=turn.tools_called,
         additionalMetadata=turn.additional_metadata,
@@ -372,17 +398,7 @@ def print_test_result(test_result: TestResult, display: TestRunResultDisplay):
     print("Metrics Summary\n")
 
     for metric_data in test_result.metrics_data:
-        successful = True
-        if metric_data.error is not None:
-            successful = False
-        else:
-            # This try block is for user defined custom metrics,
-            # which might not handle the score == undefined case elegantly
-            try:
-                if not metric_data.success:
-                    successful = False
-            except:
-                successful = False
+        successful = _is_metric_successful(metric_data)
 
         if not successful:
             print(
@@ -401,9 +417,14 @@ def print_test_result(test_result: TestResult, display: TestRunResultDisplay):
 
     elif test_result.conversational:
         print("For conversational test case:\n")
-        print(
-            f"  - Unable to print conversational test case. Run 'deepeval login' to view conversational evaluations in full."
-        )
+        if test_result.turns:
+            print("  Turns:")
+            turns = sorted(test_result.turns, key=lambda t: t.order)
+            for t in turns:
+                print(format_turn(t))
+        else:
+            print("  - No turns recorded in this test case.")
+
     else:
         print("For test case:\n")
         print(f"  - input: {test_result.input}")
@@ -470,15 +491,7 @@ def write_test_result_to_file(
         file.write("Metrics Summary\n\n")
 
         for metric_data in test_result.metrics_data:
-            successful = True
-            if metric_data.error is not None:
-                successful = False
-            else:
-                try:
-                    if not metric_data.success:
-                        successful = False
-                except:
-                    successful = False
+            successful = _is_metric_successful(metric_data)
 
             if not successful:
                 file.write(
@@ -500,9 +513,13 @@ def write_test_result_to_file(
             file.write(f"  - actual output: {test_result.actual_output}\n")
         elif test_result.conversational:
             file.write("For conversational test case:\n\n")
-            file.write(
-                "  - Unable to print conversational test case. Run 'deepeval login' to view conversational evaluations in full.\n"
-            )
+            if test_result.turns:
+                file.write("  Turns:\n")
+                turns = sorted(test_result.turns, key=lambda t: t.order)
+                for t in turns:
+                    file.write(format_turn(t) + "\n")
+            else:
+                file.write("  - No turns recorded in this test case.\n")
         else:
             file.write("For test case:\n\n")
             file.write(f"  - input: {test_result.input}\n")

deepeval-3.6.5/deepeval/integrations/crewai/__init__.py

@@ -0,0 +1,3 @@
+from .handler import instrument_crewai
+
+__all__ = ["instrument_crewai"]

deepeval-3.6.5/deepeval/integrations/crewai/handler.py

@@ -0,0 +1,196 @@
+import logging
+import deepeval
+
+from typing import Optional
+from deepeval.telemetry import capture_tracing_integration
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.tracing.tracing import Observer
+from deepeval.tracing.types import LlmSpan
+from deepeval.config.settings import get_settings
+
+
+logger = logging.getLogger(__name__)
+
+
+try:
+    from crewai.utilities.events.base_event_listener import BaseEventListener
+    from crewai.events import (
+        CrewKickoffStartedEvent,
+        CrewKickoffCompletedEvent,
+        LLMCallStartedEvent,
+        LLMCallCompletedEvent,
+        AgentExecutionStartedEvent,
+        AgentExecutionCompletedEvent,
+        ToolUsageStartedEvent,
+        ToolUsageFinishedEvent,
+    )
+
+    crewai_installed = True
+except ImportError as e:
+    if get_settings().DEEPEVAL_VERBOSE_MODE:
+        if isinstance(e, ModuleNotFoundError):
+            logger.warning(
+                "Optional crewai dependency not installed: %s",
+                e.name,
+                stacklevel=2,
+            )
+        else:
+            logger.warning(
+                "Optional crewai import failed: %s",
+                e,
+                stacklevel=2,
+            )
+
+    crewai_installed = False
+
+IS_WRAPPED_ALL = False
+
+
+def is_crewai_installed():
+    if not crewai_installed:
+        raise ImportError(
+            "CrewAI is not installed. Please install it with `pip install crewai`."
+        )
+
+
+class CrewAIEventsListener(BaseEventListener):
+    def __init__(self):
+        is_crewai_installed()
+        super().__init__()
+        self.span_observers: dict[str, Observer] = {}
+
+    @staticmethod
+    def get_tool_execution_id(source, event) -> str:
+        source_id = id(source)
+        task_id = getattr(event, "task_id", "unknown")
+        agent_id = getattr(event, "agent_id", "unknown")
+        tool_name = getattr(event, "tool_name", "unknown")
+        execution_id = f"tool_{source_id}_{task_id}_{agent_id}_{tool_name}"
+
+        return execution_id
+
+    def setup_listeners(self, crewai_event_bus):
+        @crewai_event_bus.on(CrewKickoffStartedEvent)
+        def on_crew_started(source, event: CrewKickoffStartedEvent):
+            # Assuming that this event is called in the crew.kickoff method
+            current_span = current_span_context.get()
+
+            # set the input
+            if current_span:
+                current_span.input = event.inputs
+
+            # set trace input
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.input = event.inputs
+
+        @crewai_event_bus.on(CrewKickoffCompletedEvent)
+        def on_crew_completed(source, event: CrewKickoffCompletedEvent):
+            # Assuming that this event is called in the crew.kickoff method
+            current_span = current_span_context.get()
+
+            # set the output
+            if current_span:
+                current_span.output = str(event.output)
+
+            # set trace output
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.output = str(event.output)
+
+        @crewai_event_bus.on(LLMCallStartedEvent)
+        def on_llm_started(source, event: LLMCallStartedEvent):
+            # Assuming that this event is called in the llm.call method
+            current_span = current_span_context.get()
+
+            # set the input
+            if current_span:
+                current_span.input = event.messages
+
+                # set the model
+                if isinstance(current_span, LlmSpan):
+                    current_span.model = event.model
+
+        @crewai_event_bus.on(LLMCallCompletedEvent)
+        def on_llm_completed(source, event: LLMCallCompletedEvent):
+            # Assuming that this event is called in the llm.call method
+            current_span = current_span_context.get()
+
+            # set the output
+            if current_span:
+                current_span.output = event.response
+
+        @crewai_event_bus.on(AgentExecutionStartedEvent)
+        def on_agent_started(source, event: AgentExecutionStartedEvent):
+            # Assuming that this event is called in the agent.execute_task method
+            current_span = current_span_context.get()
+
+            # set the input
+            if current_span:
+                current_span.input = event.task_prompt
+
+        @crewai_event_bus.on(AgentExecutionCompletedEvent)
+        def on_agent_completed(source, event: AgentExecutionCompletedEvent):
+            # Assuming that this event is called in the agent.execute_task method
+            current_span = current_span_context.get()
+
+            # set the output
+            if current_span:
+                current_span.output = event.output
+
+        @crewai_event_bus.on(ToolUsageStartedEvent)
+        def on_tool_started(source, event: ToolUsageStartedEvent):
+            observer = Observer(
+                span_type="tool",
+                func_name=event.tool_name,
+                function_kwargs=event.tool_args,
+            )
+            self.span_observers[self.get_tool_execution_id(source, event)] = (
+                observer
+            )
+            observer.__enter__()
+
+        @crewai_event_bus.on(ToolUsageFinishedEvent)
+        def on_tool_completed(source, event: ToolUsageFinishedEvent):
+            observer = self.span_observers.pop(
+                self.get_tool_execution_id(source, event)
+            )
+            if observer:
+                current_span = current_span_context.get()
+                if current_span:
+                    current_span.output = event.output
+                observer.__exit__(None, None, None)
+
+
+def instrument_crewai(api_key: Optional[str] = None):
+    is_crewai_installed()
+    with capture_tracing_integration("crewai"):
+        if api_key:
+            deepeval.login(api_key)
+
+        wrap_all()
+
+        CrewAIEventsListener()
+
+
+def wrap_all():
+    global IS_WRAPPED_ALL
+
+    if not IS_WRAPPED_ALL:
+        from deepeval.integrations.crewai.wrapper import (
+            wrap_crew_kickoff,
+            wrap_crew_kickoff_for_each,
+            wrap_crew_kickoff_async,
+            wrap_crew_kickoff_for_each_async,
+            wrap_llm_call,
+            wrap_agent_execute_task,
+        )
+
+        wrap_crew_kickoff()
+        wrap_crew_kickoff_for_each()
+        wrap_crew_kickoff_async()
+        wrap_crew_kickoff_for_each_async()
+        wrap_llm_call()
+        wrap_agent_execute_task()
+
+        IS_WRAPPED_ALL = True

deepeval-3.6.5/deepeval/integrations/crewai/wrapper.py

@@ -0,0 +1,87 @@
+from crewai.llm import LLM
+from crewai.crew import Crew
+from crewai.agent import Agent
+from functools import wraps
+from deepeval.tracing.tracing import Observer
+
+
+def wrap_crew_kickoff():
+    original_kickoff = Crew.kickoff
+
+    @wraps(original_kickoff)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff"):
+            result = original_kickoff(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff = wrapper
+
+
+def wrap_crew_kickoff_for_each():
+    original_kickoff_for_each = Crew.kickoff_for_each
+
+    @wraps(original_kickoff_for_each)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_for_each"):
+            result = original_kickoff_for_each(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff_for_each = wrapper
+
+
+def wrap_crew_kickoff_async():
+    original_kickoff_async = Crew.kickoff_async
+
+    @wraps(original_kickoff_async)
+    async def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_async"):
+            result = await original_kickoff_async(self, *args, **kwargs)
+
+        return result
+
+    Crew.kickoff_async = wrapper
+
+
+def wrap_crew_kickoff_for_each_async():
+    original_kickoff_for_each_async = Crew.kickoff_for_each_async
+
+    @wraps(original_kickoff_for_each_async)
+    async def wrapper(self, *args, **kwargs):
+        with Observer(span_type="crew", func_name="kickoff_for_each_async"):
+            result = await original_kickoff_for_each_async(
+                self, *args, **kwargs
+            )
+
+        return result
+
+    Crew.kickoff_for_each_async = wrapper
+
+
+def wrap_llm_call():
+    original_llm_call = LLM.call
+
+    @wraps(original_llm_call)
+    def wrapper(self, *args, **kwargs):
+        with Observer(
+            span_type="llm",
+            func_name="call",
+            observe_kwargs={"model": "temp_model"},
+        ):
+            result = original_llm_call(self, *args, **kwargs)
+        return result
+
+    LLM.call = wrapper
+
+
+def wrap_agent_execute_task():
+    original_execute_task = Agent.execute_task
+
+    @wraps(original_execute_task)
+    def wrapper(self, *args, **kwargs):
+        with Observer(span_type="agent", func_name="execute_task"):
+            result = original_execute_task(self, *args, **kwargs)
+        return result
+
+    Agent.execute_task = wrapper

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/integrations/pydantic_ai/instrumentator.py

@@ -1,7 +1,19 @@
 import json
+import logging
 import os
 from typing import Literal, Optional, List
 
+from deepeval.config.settings import get_settings
+from deepeval.confident.api import get_confident_api_key
+from deepeval.prompt import Prompt
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import Trace
+from deepeval.tracing.otel.utils import to_hex_string
+
+
+logger = logging.getLogger(__name__)
+
+
 try:
     from pydantic_ai.models.instrumented import InstrumentationSettings
     from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
@@ -11,7 +23,20 @@ try:
     )
 
     dependency_installed = True
-except:
+except ImportError as e:
+    if get_settings().DEEPEVAL_VERBOSE_MODE:
+        if isinstance(e, ModuleNotFoundError):
+            logger.warning(
+                "Optional tracing dependency not installed: %s",
+                e.name,
+                stacklevel=2,
+            )
+        else:
+            logger.warning(
+                "Optional tracing import failed: %s",
+                e,
+                stacklevel=2,
+            )
     dependency_installed = False
 
 
@@ -25,6 +50,10 @@ def is_dependency_installed():
 
 from deepeval.confident.api import get_confident_api_key
 from deepeval.prompt import Prompt
+from deepeval.tracing.otel.test_exporter import test_exporter
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import Trace
+from deepeval.tracing.otel.utils import to_hex_string
 
 # OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
 OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
@@ -37,6 +66,12 @@ class SpanInterceptor(SpanProcessor):
 
     def on_start(self, span, parent_context):
 
+        # set trace uuid
+        _current_trace_context = current_trace_context.get()
+        if _current_trace_context and isinstance(_current_trace_context, Trace):
+            _otel_trace_id = span.get_span_context().trace_id
+            _current_trace_context.uuid = to_hex_string(_otel_trace_id, 32)
+
         # set trace attributes
         if self.settings.thread_id:
             span.set_attribute(
@@ -148,8 +183,9 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         confident_prompt: Optional[Prompt] = None,
         llm_metric_collection: Optional[str] = None,
         agent_metric_collection: Optional[str] = None,
-        tool_metric_collection_map: dict = {},
+        tool_metric_collection_map: Optional[dict] = None,
         trace_metric_collection: Optional[str] = None,
+        is_test_mode: Optional[bool] = False,
     ):
         is_dependency_installed()
 
@@ -162,7 +198,7 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         ]:
             self.environment = _environment
 
-        self.tool_metric_collection_map = tool_metric_collection_map
+        self.tool_metric_collection_map = tool_metric_collection_map or {}
         self.name = name
         self.thread_id = thread_id
         self.user_id = user_id
@@ -185,12 +221,15 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         span_interceptor = SpanInterceptor(self)
         trace_provider.add_span_processor(span_interceptor)
 
-        trace_provider.add_span_processor(
-            BatchSpanProcessor(
-                OTLPSpanExporter(
-                    endpoint=OTLP_ENDPOINT,
-                    headers={"x-confident-api-key": api_key},
+        if is_test_mode:
+            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+        else:
+            trace_provider.add_span_processor(
+                BatchSpanProcessor(
+                    OTLPSpanExporter(
+                        endpoint=OTLP_ENDPOINT,
+                        headers={"x-confident-api-key": api_key},
+                    )
                 )
             )
-        )
         super().__init__(tracer_provider=trace_provider)

{deepeval-3.6.3 → deepeval-3.6.5}/deepeval/metrics/faithfulness/faithfulness.py

@@ -41,6 +41,7 @@ class FaithfulnessMetric(BaseMetric):
         strict_mode: bool = False,
         verbose_mode: bool = False,
         truths_extraction_limit: Optional[int] = None,
+        penalize_ambiguous_claims: bool = False,
         evaluation_template: Type[FaithfulnessTemplate] = FaithfulnessTemplate,
     ):
         self.threshold = 1 if strict_mode else threshold
@@ -51,6 +52,7 @@ class FaithfulnessMetric(BaseMetric):
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
         self.evaluation_template = evaluation_template
+        self.penalize_ambiguous_claims = penalize_ambiguous_claims
 
         self.truths_extraction_limit = truths_extraction_limit
         if self.truths_extraction_limit is not None:
@@ -329,6 +331,12 @@ class FaithfulnessMetric(BaseMetric):
             if verdict.verdict.strip().lower() != "no":
                 faithfulness_count += 1
 
+            if (
+                self.penalize_ambiguous_claims
+                and verdict.verdict.strip().lower() == "idk"
+            ):
+                faithfulness_count -= 1
+
         score = faithfulness_count / number_of_verdicts
         return 0 if self.strict_mode and score < self.threshold else score