deepeval 3.6.8__py3-none-any.whl → 3.7.0__py3-none-any.whl

deepeval/integrations/crewai/handler.py

@@ -23,6 +23,8 @@ try:
         AgentExecutionCompletedEvent,
         ToolUsageStartedEvent,
         ToolUsageFinishedEvent,
+        KnowledgeRetrievalStartedEvent,
+        KnowledgeRetrievalCompletedEvent,
     )
 
     crewai_installed = True
@@ -69,6 +71,14 @@ class CrewAIEventsListener(BaseEventListener):
 
         return execution_id
 
+    @staticmethod
+    def get_knowledge_execution_id(source, event) -> str:
+        source_id = id(source)
+        agent_id = id(event.agent) if hasattr(event, "agent") else "unknown"
+        execution_id = f"_knowledge_{source_id}_{agent_id}"
+
+        return execution_id
+
     def setup_listeners(self, crewai_event_bus):
         @crewai_event_bus.on(CrewKickoffStartedEvent)
         def on_crew_started(source, event: CrewKickoffStartedEvent):
@@ -161,6 +171,32 @@ class CrewAIEventsListener(BaseEventListener):
                     current_span.output = event.output
                 observer.__exit__(None, None, None)
 
+        @crewai_event_bus.on(KnowledgeRetrievalStartedEvent)
+        def on_knowledge_started(source, event: KnowledgeRetrievalStartedEvent):
+            observer = Observer(
+                span_type="tool",
+                func_name="knowledge_retrieval",
+                function_kwargs={},
+            )
+            self.span_observers[
+                self.get_knowledge_execution_id(source, event)
+            ] = observer
+            observer.__enter__()
+
+        @crewai_event_bus.on(KnowledgeRetrievalCompletedEvent)
+        def on_knowledge_completed(
+            source, event: KnowledgeRetrievalCompletedEvent
+        ):
+            observer = self.span_observers.pop(
+                self.get_knowledge_execution_id(source, event)
+            )
+            if observer:
+                current_span = current_span_context.get()
+                if current_span:
+                    current_span.input = event.query
+                    current_span.output = event.retrieved_knowledge
+                observer.__exit__(None, None, None)
+
 
 def instrument_crewai(api_key: Optional[str] = None):
     is_crewai_installed()

deepeval/integrations/langchain/callback.py

@@ -1,12 +1,15 @@
 from typing import Any, Optional, List, Dict
 from uuid import UUID
 from time import perf_counter
-from deepeval.tracing.context import current_trace_context
+
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.test_case.llm_test_case import ToolCall
 from deepeval.tracing.types import (
     LlmOutput,
     LlmToolCall,
 )
 from deepeval.metrics import BaseMetric
+from deepeval.tracing.utils import prepare_tool_call_input_parameters
 
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
@@ -266,12 +269,34 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
-
         uuid_str = str(run_id)
         tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
         tool_span.output = output
         exit_current_context(uuid_str=uuid_str)
 
+        # set the tools called in the parent span as well as on the trace level
+        tool_call = ToolCall(
+            name=tool_span.name,
+            description=tool_span.description,
+            output=output,
+            input_parameters=prepare_tool_call_input_parameters(
+                tool_span.input
+            ),
+        )
+        parent_span = current_span_context.get()
+        if parent_span:
+            if parent_span.tools_called is None:
+                parent_span.tools_called = []
+
+            parent_span.tools_called.append(tool_call)
+
+        trace = current_trace_context.get()
+        if trace:
+            if trace.tools_called is None:
+                trace.tools_called = []
+
+            trace.tools_called.append(tool_call)
+
     def on_tool_error(
         self,
         error: BaseException,

deepeval/integrations/llama_index/handler.py

@@ -2,13 +2,28 @@ from typing import Any, Dict, Optional
 import inspect
 from time import perf_counter
 import uuid
+
+from llama_index.core.agent.workflow.workflow_events import (
+    AgentWorkflowStartEvent,
+)
+from deepeval.integrations.llama_index.utils import (
+    extract_output_from_llm_chat_end_event,
+)
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing import trace_manager
-from deepeval.tracing.types import AgentSpan, BaseSpan, LlmSpan, TraceSpanStatus
+from deepeval.tracing.types import (
+    ToolSpan,
+    AgentSpan,
+    BaseSpan,
+    LlmSpan,
+    TraceSpanStatus,
+)
 from deepeval.tracing.trace_context import (
     current_llm_context,
     current_agent_context,
 )
+from deepeval.test_case import ToolCall
+from deepeval.tracing.utils import make_json_serializable
 
 try:
     from llama_index.core.instrumentation.events.base import BaseEvent
@@ -89,6 +104,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     if llm_span_context
                     else None
                 ),
+                prompt=llm_span_context.prompt if llm_span_context else None,
             )
             trace_manager.add_span(llm_span)
             trace_manager.add_span_to_trace(llm_span)
@@ -106,7 +122,9 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     llm_span.status = TraceSpanStatus.SUCCESS
                     llm_span.end_time = perf_counter()
                     llm_span.input = llm_span.input
-                    llm_span.output = event.response.message.blocks[0].text
+                    llm_span.output = extract_output_from_llm_chat_end_event(
+                        event
+                    )
                     trace_manager.remove_span(llm_span.uuid)
                     del self.open_ai_astream_to_llm_span_map[event.span_id]
 
@@ -151,6 +169,14 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
         # conditions to qualify as agent start run span
         if method_name == "run":
             agent_span_context = current_agent_context.get()
+            start_event = bound_args.arguments.get("start_event")
+
+            if start_event and isinstance(start_event, AgentWorkflowStartEvent):
+                input = start_event.model_dump()
+
+            else:
+                input = bound_args.arguments
+
             span = AgentSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -159,7 +185,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 parent_uuid=parent_span_id,
                 start_time=perf_counter(),
                 name="Agent",  # TODO: decide the name of the span
-                input=bound_args.arguments,
+                input=input,
                 metrics=(
                     agent_span_context.metrics if agent_span_context else None
                 ),
@@ -169,7 +195,17 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     else None
                 ),
             )
-
+        elif method_name == "acall":
+            span = ToolSpan(
+                uuid=id_,
+                status=TraceSpanStatus.IN_PROGRESS,
+                children=[],
+                trace_uuid=trace_uuid,
+                parent_uuid=parent_span_id,
+                start_time=perf_counter(),
+                input=bound_args.arguments,
+                name="Tool",
+            )
         # prepare input test case params for the span
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments
@@ -192,10 +228,28 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
         if base_span is None:
             return None
 
+        class_name, method_name = parse_id(id_)
+        if method_name == "call_tool":
+            output_json = make_json_serializable(result)
+            if output_json and isinstance(output_json, dict):
+                if base_span.tools_called is None:
+                    base_span.tools_called = []
+                base_span.tools_called.append(
+                    ToolCall(
+                        name=output_json.get("tool_name", "Tool"),
+                        input_parameters=output_json.get("tool_kwargs", {}),
+                        output=output_json.get("tool_output", {}),
+                    )
+                )
         base_span.end_time = perf_counter()
         base_span.status = TraceSpanStatus.SUCCESS
         base_span.output = result
 
+        if isinstance(base_span, ToolSpan):
+            result_json = make_json_serializable(result)
+            if result_json and isinstance(result_json, dict):
+                base_span.name = result_json.get("tool_name", "Tool")
+
         if base_span.llm_test_case:
             class_name, method_name = parse_id(id_)
             prepare_output_llm_test_case_params(

deepeval/integrations/llama_index/utils.py

@@ -1,3 +1,4 @@
+from llama_index.core.instrumentation.events.llm import LLMChatEndEvent
 from deepeval.test_case.llm_test_case import LLMTestCase, ToolCall
 from deepeval.tracing.types import BaseSpan
 from typing import Any
@@ -81,3 +82,26 @@ def prepare_output_llm_test_case_params(
                 )
 
             span.llm_test_case.tools_called = tool_calls
+
+
+def extract_output_from_llm_chat_end_event(event: LLMChatEndEvent) -> list:
+    messages = []
+    for msg in event.response.message.blocks:
+        if msg.block_type == "text":
+            messages.append(
+                {
+                    "role": event.response.message.role.value,
+                    "content": msg.text,
+                }
+            )
+        elif msg.block_type == "tool_call":
+            messages.append(
+                {
+                    "name": msg.tool_name,
+                    "input_parameters": msg.tool_kwargs,
+                    "id": msg.tool_call_id,
+                }
+            )
+        else:
+            messages.append(msg.model_dump())
+    return messages

deepeval/metrics/__init__.py

@@ -8,6 +8,8 @@ from .base_metric import (
 from .dag.dag import DAGMetric, DeepAcyclicGraph
 from .conversational_dag.conversational_dag import ConversationalDAGMetric
 from .bias.bias import BiasMetric
+from .exact_match.exact_match import ExactMatchMetric
+from .pattern_match.pattern_match import PatternMatchMetric
 from .toxicity.toxicity import ToxicityMetric
 from .pii_leakage.pii_leakage import PIILeakageMetric
 from .non_advice.non_advice import NonAdviceMetric
@@ -69,6 +71,9 @@ __all__ = [
     "BaseConversationalMetric",
     "BaseMultimodalMetric",
     "BaseArenaMetric",
+    # Non-LLM metrics
+    "ExactMatchMetric",
+    "PatternMatchMetric",
     # Core metrics
     "GEval",
     "ArenaGEval",

deepeval/metrics/exact_match/exact_match.py

@@ -0,0 +1,94 @@
+from typing import List
+
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    check_llm_test_case_params,
+    construct_verbose_logs,
+)
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+
+class ExactMatchMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.EXPECTED_OUTPUT,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 1,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = threshold
+        self.verbose_mode = verbose_mode
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            expected = test_case.expected_output.strip()
+            actual = test_case.actual_output.strip()
+
+            if expected == actual:
+                self.score = self.precision = self.recall = self.f1 = 1.0
+                self.reason = (
+                    "The actual and expected outputs are exact matches."
+                )
+            else:
+                self.score = self.precision = self.recall = self.f1 = 0.0
+                self.reason = "The actual and expected outputs are different."
+
+            self.success = self.score >= self.threshold
+
+            if self.verbose_mode:
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Score: {self.score:.2f}",
+                        f"Reason: {self.reason}",
+                    ],
+                )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        return self.measure(
+            test_case,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        )
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Exact Match"

deepeval/metrics/indicator.py

@@ -18,6 +18,10 @@ from deepeval.test_run.cache import CachedTestCase, Cache
 from deepeval.telemetry import capture_metric_type
 from deepeval.utils import update_pbar
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 def format_metric_description(
     metric: Union[BaseMetric, BaseConversationalMetric, BaseArenaMetric],
@@ -43,7 +47,7 @@ def metric_progress_indicator(
     _show_indicator: bool = True,
     _in_component: bool = False,
 ):
-    captured_async_mode = False if async_mode == None else async_mode
+    captured_async_mode = False if async_mode is None else async_mode
     with capture_metric_type(
         metric.__name__,
         async_mode=captured_async_mode,
@@ -250,6 +254,21 @@ async def safe_a_measure(
             _log_metric_to_confident=False,
         )
         update_pbar(progress, pbar_eval_id)
+
+    except asyncio.CancelledError:
+        logger.info("caught asyncio.CancelledError")
+
+        # treat cancellation as a timeout so we still emit a MetricData
+        metric.error = (
+            "Timed out/cancelled while evaluating metric. "
+            "Increase DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE or set "
+            "DEEPEVAL_LOG_STACK_TRACES=1 for full traceback."
+        )
+        metric.success = False
+
+        if not ignore_errors:
+            raise
+
     except MissingTestCaseParamsError as e:
         if skip_on_missing_params:
             metric.skipped = True
@@ -277,5 +296,6 @@ async def safe_a_measure(
         if ignore_errors:
             metric.error = str(e)
             metric.success = False  # Assuming you want to set success to False
+            logger.info("a metric was marked as errored")
         else:
             raise

deepeval/metrics/pattern_match/pattern_match.py

@@ -0,0 +1,103 @@
+import re
+from typing import List
+
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    check_llm_test_case_params,
+    construct_verbose_logs,
+)
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+
+class PatternMatchMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ]
+
+    def __init__(
+        self,
+        pattern: str,
+        ignore_case: bool = False,
+        threshold: float = 1.0,
+        verbose_mode: bool = False,
+    ):
+        self.pattern = pattern.strip()
+        self.ignore_case = ignore_case
+        self.verbose_mode = verbose_mode
+        self.threshold = threshold
+
+        flags = re.IGNORECASE if ignore_case else 0
+        try:
+            self._compiled_pattern = re.compile(self.pattern, flags)
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern: {pattern} — {e}")
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            actual = test_case.actual_output.strip()
+            full_match = self._compiled_pattern.fullmatch(actual)
+
+            self.score = 1.0 if full_match else 0.0
+            self.reason = (
+                f"The actual output fully matches the pattern."
+                if full_match
+                else f"The actual output does not match the pattern."
+            )
+            self.success = self.score >= self.threshold
+
+            if self.verbose_mode:
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Pattern: {self.pattern}",
+                        f"Actual: {actual}",
+                        f"Score: {self.score:.2f}",
+                        f"Reason: {self.reason}",
+                    ],
+                )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        return self.measure(
+            test_case,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        )
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Pattern Match"

deepeval/metrics/task_completion/task_completion.py

@@ -36,6 +36,11 @@ class TaskCompletionMetric(BaseMetric):
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        if task is None:
+            self._is_task_provided = False
+        else:
+            self._is_task_provided = True
+
         self.task = task
         self.threshold = 1 if strict_mode else threshold
         self.model, self.using_native_model = initialize_model(model)
@@ -73,7 +78,8 @@ class TaskCompletionMetric(BaseMetric):
                 )
             else:
                 task, self.outcome = self._extract_task_and_outcome(test_case)
-                self.task = task if self.task is None else self.task
+                if self.task is None or not self._is_task_provided:
+                    self.task = task
                 self.verdict, self.reason = self._generate_verdicts()
                 self.score = self._calculate_score()
                 self.success = self.score >= self.threshold
@@ -108,7 +114,8 @@ class TaskCompletionMetric(BaseMetric):
             task, self.outcome = await self._a_extract_task_and_outcome(
                 test_case
             )
-            self.task = task if self.task is None else self.task
+            if self.task is None or not self._is_task_provided:
+                self.task = task
             self.verdict, self.reason = await self._a_generate_verdicts()
             self.score = self._calculate_score()
             self.success = self.score >= self.threshold

deepeval/model_integrations/utils.py

@@ -0,0 +1,116 @@
+import json
+import uuid
+from typing import Any, List, Optional
+
+from deepeval.model_integrations.types import InputParameters, OutputParameters
+from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import (
+    current_span_context,
+    current_trace_context,
+    update_current_span,
+    update_llm_span,
+)
+from deepeval.tracing.trace_context import current_llm_context
+from deepeval.tracing.types import ToolSpan, TraceSpanStatus
+from deepeval.utils import shorten, len_long
+
+
+def _update_all_attributes(
+    input_parameters: InputParameters,
+    output_parameters: OutputParameters,
+    expected_tools: List[ToolCall],
+    expected_output: str,
+    context: List[str],
+    retrieval_context: List[str],
+):
+    """Update span and trace attributes with input/output parameters."""
+    update_current_span(
+        input=input_parameters.input or input_parameters.messages or "NA",
+        output=output_parameters.output or "NA",
+        tools_called=output_parameters.tools_called,
+        # attributes to be added
+        expected_output=expected_output,
+        expected_tools=expected_tools,
+        context=context,
+        retrieval_context=retrieval_context,
+    )
+
+    llm_context = current_llm_context.get()
+
+    update_llm_span(
+        input_token_count=output_parameters.prompt_tokens,
+        output_token_count=output_parameters.completion_tokens,
+        prompt=llm_context.prompt,
+    )
+
+    if output_parameters.tools_called:
+        create_child_tool_spans(output_parameters)
+
+    __update_input_and_output_of_current_trace(
+        input_parameters, output_parameters
+    )
+
+
+def __update_input_and_output_of_current_trace(
+    input_parameters: InputParameters, output_parameters: OutputParameters
+):
+
+    current_trace = current_trace_context.get()
+    if current_trace:
+        if current_trace.input is None:
+            current_trace.input = (
+                input_parameters.input or input_parameters.messages
+            )
+        if current_trace.output is None:
+            current_trace.output = output_parameters.output
+
+    return
+
+
+def create_child_tool_spans(output_parameters: OutputParameters):
+    if output_parameters.tools_called is None:
+        return
+
+    current_span = current_span_context.get()
+    for tool_called in output_parameters.tools_called:
+        tool_span = ToolSpan(
+            **{
+                "uuid": str(uuid.uuid4()),
+                "trace_uuid": current_span.trace_uuid,
+                "parent_uuid": current_span.uuid,
+                "start_time": current_span.start_time,
+                "end_time": current_span.start_time,
+                "status": TraceSpanStatus.SUCCESS,
+                "children": [],
+                "name": tool_called.name,
+                "input": tool_called.input_parameters,
+                "output": None,
+                "metrics": None,
+                "description": tool_called.description,
+            }
+        )
+        current_span.children.append(tool_span)
+
+
+_URL_MAX = 200
+_JSON_MAX = max(
+    len_long(), 400
+)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
+
+
+def compact_dump(value: Any) -> str:
+    try:
+        dumped = json.dumps(
+            value, ensure_ascii=False, default=str, separators=(",", ":")
+        )
+    except Exception:
+        dumped = repr(value)
+    return shorten(dumped, max_len=_JSON_MAX)
+
+
+def fmt_url(url: Optional[str]) -> str:
+    if not url:
+        return ""
+    if url.startswith("data:"):
+        return "[data-uri]"
+    return shorten(url, max_len=_URL_MAX)

deepeval/models/base_model.py

@@ -68,7 +68,9 @@ class DeepEvalBaseLLM(ABC):
         Returns:
             A list of strings.
         """
-        raise AttributeError
+        raise NotImplementedError(
+            "batch_generate is not implemented for this model"
+        )
 
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str: