deepeval 3.6.9__py3-none-any.whl → 3.7.1__py3-none-any.whl

deepeval/integrations/langchain/callback.py

@@ -1,12 +1,15 @@
 from typing import Any, Optional, List, Dict
 from uuid import UUID
 from time import perf_counter
-from deepeval.tracing.context import current_trace_context
+
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.test_case.llm_test_case import ToolCall
 from deepeval.tracing.types import (
     LlmOutput,
     LlmToolCall,
 )
 from deepeval.metrics import BaseMetric
+from deepeval.tracing.utils import prepare_tool_call_input_parameters
 
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
@@ -266,12 +269,34 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
-
         uuid_str = str(run_id)
         tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
         tool_span.output = output
         exit_current_context(uuid_str=uuid_str)
 
+        # set the tools called in the parent span as well as on the trace level
+        tool_call = ToolCall(
+            name=tool_span.name,
+            description=tool_span.description,
+            output=output,
+            input_parameters=prepare_tool_call_input_parameters(
+                tool_span.input
+            ),
+        )
+        parent_span = current_span_context.get()
+        if parent_span:
+            if parent_span.tools_called is None:
+                parent_span.tools_called = []
+
+            parent_span.tools_called.append(tool_call)
+
+        trace = current_trace_context.get()
+        if trace:
+            if trace.tools_called is None:
+                trace.tools_called = []
+
+            trace.tools_called.append(tool_call)
+
     def on_tool_error(
         self,
         error: BaseException,

deepeval/integrations/llama_index/handler.py

@@ -2,13 +2,28 @@ from typing import Any, Dict, Optional
 import inspect
 from time import perf_counter
 import uuid
+
+from llama_index.core.agent.workflow.workflow_events import (
+    AgentWorkflowStartEvent,
+)
+from deepeval.integrations.llama_index.utils import (
+    extract_output_from_llm_chat_end_event,
+)
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing import trace_manager
-from deepeval.tracing.types import AgentSpan, BaseSpan, LlmSpan, TraceSpanStatus
+from deepeval.tracing.types import (
+    ToolSpan,
+    AgentSpan,
+    BaseSpan,
+    LlmSpan,
+    TraceSpanStatus,
+)
 from deepeval.tracing.trace_context import (
     current_llm_context,
     current_agent_context,
 )
+from deepeval.test_case import ToolCall
+from deepeval.tracing.utils import make_json_serializable
 
 try:
     from llama_index.core.instrumentation.events.base import BaseEvent
@@ -89,6 +104,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     if llm_span_context
                     else None
                 ),
+                prompt=llm_span_context.prompt if llm_span_context else None,
             )
             trace_manager.add_span(llm_span)
             trace_manager.add_span_to_trace(llm_span)
@@ -106,7 +122,9 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     llm_span.status = TraceSpanStatus.SUCCESS
                     llm_span.end_time = perf_counter()
                     llm_span.input = llm_span.input
-                    llm_span.output = event.response.message.blocks[0].text
+                    llm_span.output = extract_output_from_llm_chat_end_event(
+                        event
+                    )
                     trace_manager.remove_span(llm_span.uuid)
                     del self.open_ai_astream_to_llm_span_map[event.span_id]
 
@@ -151,6 +169,14 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
         # conditions to qualify as agent start run span
         if method_name == "run":
             agent_span_context = current_agent_context.get()
+            start_event = bound_args.arguments.get("start_event")
+
+            if start_event and isinstance(start_event, AgentWorkflowStartEvent):
+                input = start_event.model_dump()
+
+            else:
+                input = bound_args.arguments
+
             span = AgentSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -159,7 +185,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 parent_uuid=parent_span_id,
                 start_time=perf_counter(),
                 name="Agent",  # TODO: decide the name of the span
-                input=bound_args.arguments,
+                input=input,
                 metrics=(
                     agent_span_context.metrics if agent_span_context else None
                 ),
@@ -169,7 +195,17 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     else None
                 ),
             )
-
+        elif method_name == "acall":
+            span = ToolSpan(
+                uuid=id_,
+                status=TraceSpanStatus.IN_PROGRESS,
+                children=[],
+                trace_uuid=trace_uuid,
+                parent_uuid=parent_span_id,
+                start_time=perf_counter(),
+                input=bound_args.arguments,
+                name="Tool",
+            )
         # prepare input test case params for the span
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments
@@ -192,10 +228,28 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
         if base_span is None:
             return None
 
+        class_name, method_name = parse_id(id_)
+        if method_name == "call_tool":
+            output_json = make_json_serializable(result)
+            if output_json and isinstance(output_json, dict):
+                if base_span.tools_called is None:
+                    base_span.tools_called = []
+                base_span.tools_called.append(
+                    ToolCall(
+                        name=output_json.get("tool_name", "Tool"),
+                        input_parameters=output_json.get("tool_kwargs", {}),
+                        output=output_json.get("tool_output", {}),
+                    )
+                )
         base_span.end_time = perf_counter()
         base_span.status = TraceSpanStatus.SUCCESS
         base_span.output = result
 
+        if isinstance(base_span, ToolSpan):
+            result_json = make_json_serializable(result)
+            if result_json and isinstance(result_json, dict):
+                base_span.name = result_json.get("tool_name", "Tool")
+
         if base_span.llm_test_case:
             class_name, method_name = parse_id(id_)
             prepare_output_llm_test_case_params(

deepeval/integrations/llama_index/utils.py

@@ -1,3 +1,4 @@
+from llama_index.core.instrumentation.events.llm import LLMChatEndEvent
 from deepeval.test_case.llm_test_case import LLMTestCase, ToolCall
 from deepeval.tracing.types import BaseSpan
 from typing import Any
@@ -81,3 +82,26 @@ def prepare_output_llm_test_case_params(
                 )
 
             span.llm_test_case.tools_called = tool_calls
+
+
+def extract_output_from_llm_chat_end_event(event: LLMChatEndEvent) -> list:
+    messages = []
+    for msg in event.response.message.blocks:
+        if msg.block_type == "text":
+            messages.append(
+                {
+                    "role": event.response.message.role.value,
+                    "content": msg.text,
+                }
+            )
+        elif msg.block_type == "tool_call":
+            messages.append(
+                {
+                    "name": msg.tool_name,
+                    "input_parameters": msg.tool_kwargs,
+                    "id": msg.tool_call_id,
+                }
+            )
+        else:
+            messages.append(msg.model_dump())
+    return messages

deepeval/key_handler.py

@@ -70,6 +70,7 @@ class ModelKeyValues(Enum):
     GOOGLE_GENAI_USE_VERTEXAI = "GOOGLE_GENAI_USE_VERTEXAI"
     GOOGLE_CLOUD_PROJECT = "GOOGLE_CLOUD_PROJECT"
     GOOGLE_CLOUD_LOCATION = "GOOGLE_CLOUD_LOCATION"
+    GOOGLE_SERVICE_ACCOUNT_KEY = "GOOGLE_SERVICE_ACCOUNT_KEY"
     # LiteLLM
     USE_LITELLM = "USE_LITELLM"
     LITELLM_MODEL_NAME = "LITELLM_MODEL_NAME"

deepeval/metrics/__init__.py

@@ -8,6 +8,8 @@ from .base_metric import (
 from .dag.dag import DAGMetric, DeepAcyclicGraph
 from .conversational_dag.conversational_dag import ConversationalDAGMetric
 from .bias.bias import BiasMetric
+from .exact_match.exact_match import ExactMatchMetric
+from .pattern_match.pattern_match import PatternMatchMetric
 from .toxicity.toxicity import ToxicityMetric
 from .pii_leakage.pii_leakage import PIILeakageMetric
 from .non_advice.non_advice import NonAdviceMetric
@@ -69,6 +71,9 @@ __all__ = [
     "BaseConversationalMetric",
     "BaseMultimodalMetric",
     "BaseArenaMetric",
+    # Non-LLM metrics
+    "ExactMatchMetric",
+    "PatternMatchMetric",
     # Core metrics
     "GEval",
     "ArenaGEval",

deepeval/metrics/arena_g_eval/arena_g_eval.py

@@ -46,7 +46,11 @@ class ArenaGEval(BaseArenaMetric):
         self.criteria = criteria
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
-        self.evaluation_steps = evaluation_steps
+        self.evaluation_steps = (
+            evaluation_steps
+            if evaluation_steps and len(evaluation_steps) > 0
+            else None
+        )
         self.async_mode = async_mode
         self.verbose_mode = verbose_mode
         self._include_g_eval_suffix = _include_g_eval_suffix

deepeval/metrics/arena_g_eval/utils.py

@@ -89,10 +89,10 @@ class FormattedArenaTestCase:
 def format_arena_test_case(
     evaluation_params: List[LLMTestCaseParams], test_case: ArenaTestCase
 ) -> Tuple[FormattedArenaTestCase, Dict[str, str]]:
-    case = next(iter(test_case.contestants.values()))
+    case = next(iter([case.test_case for case in test_case.contestants]))
 
     # Create dummy name mapping
-    real_names = list(test_case.contestants.keys())
+    real_names = list([case.name for case in test_case.contestants])
     available_fake_names = FAKE_NAMES.copy()
     random.shuffle(available_fake_names)
 
@@ -119,10 +119,10 @@ def format_arena_test_case(
             else None
         ),
         contestants={
-            contestant: construct_formatted_llm_test_case(
-                evaluation_params, test_case
+            contestant.name: construct_formatted_llm_test_case(
+                evaluation_params, contestant.test_case
             )
-            for contestant, test_case in test_case.contestants.items()
+            for contestant in test_case.contestants
         },
         dummy_to_real_names=dummy_to_real_names,
     )

deepeval/metrics/conversational_g_eval/conversational_g_eval.py

@@ -9,6 +9,8 @@ from deepeval.metrics.g_eval.utils import (
     construct_conversational_g_eval_turn_params_string,
     construct_non_turns_test_case_string,
     format_rubrics,
+    validate_and_sort_rubrics,
+    validate_criteria_and_evaluation_steps,
 )
 from deepeval.test_case import (
     TurnParams,
@@ -63,27 +65,16 @@ class ConversationalGEval(BaseConversationalMetric):
 
         self.evaluation_params = evaluation_params
 
-        # Check if both criteria and evaluation_steps are not None at the same time
-        if criteria is None and evaluation_steps is None:
-            raise ValueError(
-                "Either 'criteria' or 'evaluation_steps' must be provided."
-            )
-
-        # Check if criteria is provided, it cannot be an empty string
-        if criteria is not None and not criteria.strip():
-            raise ValueError("Criteria provided cannot be an empty string.")
-
-        # Check if evaluation_steps is provided, it cannot be an empty list
-        if evaluation_steps is not None and len(evaluation_steps) == 0:
-            raise ValueError(
-                "'evaluation_steps' must not be an empty list. Either omit evaluation steps or include a non-empty list of steps."
-            )
-
+        validate_criteria_and_evaluation_steps(criteria, evaluation_steps)
         self.criteria = criteria
-        self.rubric = rubric
+        self.rubric = validate_and_sort_rubrics(rubric)
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
-        self.evaluation_steps = evaluation_steps
+        self.evaluation_steps = (
+            evaluation_steps
+            if evaluation_steps and len(evaluation_steps) > 0
+            else None
+        )
         self.threshold = 1 if strict_mode else threshold
         self.strict_mode = strict_mode
         self.async_mode = async_mode

deepeval/metrics/exact_match/exact_match.py

@@ -0,0 +1,94 @@
+from typing import List
+
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    check_llm_test_case_params,
+    construct_verbose_logs,
+)
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+
+class ExactMatchMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.EXPECTED_OUTPUT,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 1,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = threshold
+        self.verbose_mode = verbose_mode
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            expected = test_case.expected_output.strip()
+            actual = test_case.actual_output.strip()
+
+            if expected == actual:
+                self.score = self.precision = self.recall = self.f1 = 1.0
+                self.reason = (
+                    "The actual and expected outputs are exact matches."
+                )
+            else:
+                self.score = self.precision = self.recall = self.f1 = 0.0
+                self.reason = "The actual and expected outputs are different."
+
+            self.success = self.score >= self.threshold
+
+            if self.verbose_mode:
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Score: {self.score:.2f}",
+                        f"Reason: {self.reason}",
+                    ],
+                )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        return self.measure(
+            test_case,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        )
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Exact Match"

deepeval/metrics/g_eval/g_eval.py

@@ -61,7 +61,11 @@ class GEval(BaseMetric):
         self.score_range_span = self.score_range[1] - self.score_range[0]
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
-        self.evaluation_steps = evaluation_steps
+        self.evaluation_steps = (
+            evaluation_steps
+            if evaluation_steps and len(evaluation_steps) > 0
+            else None
+        )
         self.threshold = 1 if strict_mode else threshold
         self.top_logprobs = top_logprobs
         self.strict_mode = strict_mode

deepeval/metrics/g_eval/utils.py

@@ -77,7 +77,7 @@ def validate_criteria_and_evaluation_steps(
 def validate_and_sort_rubrics(
     rubrics: Optional[List[Rubric]] = None,
 ) -> Optional[List[Rubric]]:
-    if rubrics is None:
+    if rubrics is None or len(rubrics) == 0:
         return None
 
     # Sort rubrics by start of range

deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py

@@ -64,7 +64,11 @@ class MultimodalGEval(BaseMultimodalMetric):
         self.rubric = validate_and_sort_rubrics(rubric)
         self.model, self.using_native_model = initialize_multimodal_model(model)
         self.evaluation_model = self.model.get_model_name()
-        self.evaluation_steps = evaluation_steps
+        self.evaluation_steps = (
+            evaluation_steps
+            if evaluation_steps and len(evaluation_steps) > 0
+            else None
+        )
         self.threshold = 1 if strict_mode else threshold
         self.top_logprobs = top_logprobs
         self.strict_mode = strict_mode

deepeval/metrics/pattern_match/pattern_match.py

@@ -0,0 +1,103 @@
+import re
+from typing import List
+
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    check_llm_test_case_params,
+    construct_verbose_logs,
+)
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+
+class PatternMatchMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ]
+
+    def __init__(
+        self,
+        pattern: str,
+        ignore_case: bool = False,
+        threshold: float = 1.0,
+        verbose_mode: bool = False,
+    ):
+        self.pattern = pattern.strip()
+        self.ignore_case = ignore_case
+        self.verbose_mode = verbose_mode
+        self.threshold = threshold
+
+        flags = re.IGNORECASE if ignore_case else 0
+        try:
+            self._compiled_pattern = re.compile(self.pattern, flags)
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern: {pattern} — {e}")
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            actual = test_case.actual_output.strip()
+            full_match = self._compiled_pattern.fullmatch(actual)
+
+            self.score = 1.0 if full_match else 0.0
+            self.reason = (
+                f"The actual output fully matches the pattern."
+                if full_match
+                else f"The actual output does not match the pattern."
+            )
+            self.success = self.score >= self.threshold
+
+            if self.verbose_mode:
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Pattern: {self.pattern}",
+                        f"Actual: {actual}",
+                        f"Score: {self.score:.2f}",
+                        f"Reason: {self.reason}",
+                    ],
+                )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        return self.measure(
+            test_case,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        )
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Pattern Match"

deepeval/metrics/task_completion/task_completion.py

@@ -36,6 +36,11 @@ class TaskCompletionMetric(BaseMetric):
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        if task is None:
+            self._is_task_provided = False
+        else:
+            self._is_task_provided = True
+
         self.task = task
         self.threshold = 1 if strict_mode else threshold
         self.model, self.using_native_model = initialize_model(model)
@@ -73,7 +78,8 @@ class TaskCompletionMetric(BaseMetric):
                 )
             else:
                 task, self.outcome = self._extract_task_and_outcome(test_case)
-                self.task = task if self.task is None else self.task
+                if self.task is None or not self._is_task_provided:
+                    self.task = task
                 self.verdict, self.reason = self._generate_verdicts()
                 self.score = self._calculate_score()
                 self.success = self.score >= self.threshold
@@ -108,7 +114,8 @@ class TaskCompletionMetric(BaseMetric):
             task, self.outcome = await self._a_extract_task_and_outcome(
                 test_case
             )
-            self.task = task if self.task is None else self.task
+            if self.task is None or not self._is_task_provided:
+                self.task = task
             self.verdict, self.reason = await self._a_generate_verdicts()
             self.score = self._calculate_score()
             self.success = self.score >= self.threshold

deepeval/metrics/utils.py

@@ -270,7 +270,7 @@ def check_arena_test_case_params(
             f"Expected ArenaTestCase, got {type(arena_test_case).__name__}"
         )
 
-    cases = list(arena_test_case.contestants.values())
+    cases = [contestant.test_case for contestant in arena_test_case.contestants]
     ref_input = cases[0].input
     for case in cases[1:]:
         if case.input != ref_input: