deepeval 3.6.6__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_faithfulness/multimodal_faithfulness.py

@@ -53,6 +53,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -71,6 +72,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -97,6 +99,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py

@@ -78,6 +78,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
 
@@ -96,6 +97,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                         _additional_context=_additional_context,
                     )
                 )
@@ -132,6 +134,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         _show_indicator: bool = True,
         _in_component: bool = False,
         _additional_context: Optional[str] = None,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_mllm_test_case_params(

deepeval/metrics/multimodal_metrics/multimodal_tool_correctness/multimodal_tool_correctness.py

@@ -3,7 +3,7 @@ from typing import List, Dict
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    check_llm_test_case_params,
+    check_mllm_test_case_params,
 )
 from deepeval.test_case import (
     MLLMTestCase,
@@ -11,10 +11,10 @@ from deepeval.test_case import (
     ToolCallParams,
     ToolCall,
 )
-from deepeval.metrics import BaseMetric
+from deepeval.metrics import BaseMultimodalMetric
 
 
-class MultimodalToolCorrectnessMetric(BaseMetric):
+class MultimodalToolCorrectnessMetric(BaseMultimodalMetric):
 
     _required_params: List[MLLMTestCaseParams] = [
         MLLMTestCaseParams.INPUT,
@@ -46,8 +46,11 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_mllm_test_case_params(
+            test_case, self._required_params, None, None, self
+        )
         self.test_case = test_case
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component
@@ -90,11 +93,13 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         return self.measure(
             test_case,
             _show_indicator=_show_indicator,
             _in_component=_in_component,
+            _log_metric_to_confident=_log_metric_to_confident,
         )
 
     ##################################################
@@ -278,7 +283,7 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
 
     @property
     def __name__(self):
-        return "Tool Correctness"
+        return "Multi Modal Tool Correctness"
 
     def indent_multiline_string(self, s, indent_level=4):
         indent = " " * indent_level

deepeval/metrics/non_advice/non_advice.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.non_advice.template import NonAdviceTemplate
 from deepeval.metrics.non_advice.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class NonAdviceMetric(BaseMetric):
@@ -58,6 +59,7 @@ class NonAdviceMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -73,6 +75,7 @@ class NonAdviceMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -93,6 +96,10 @@ class NonAdviceMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -101,6 +108,7 @@ class NonAdviceMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -129,6 +137,10 @@ class NonAdviceMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.pii_leakage.template import PIILeakageTemplate
 from deepeval.metrics.pii_leakage.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class PIILeakageMetric(BaseMetric):
@@ -49,6 +50,7 @@ class PIILeakageMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -64,6 +66,7 @@ class PIILeakageMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class PIILeakageMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -92,6 +99,7 @@ class PIILeakageMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -120,7 +128,10 @@ class PIILeakageMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/prompt_alignment/prompt_alignment.py

@@ -20,6 +20,8 @@ from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.prompt_alignment import schema as paschema
 from deepeval.config.settings import get_settings
 
+from deepeval.metrics.api import metric_data_manager
+
 
 class PromptAlignmentMetric(BaseMetric):
 
@@ -55,6 +57,7 @@ class PromptAlignmentMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -93,6 +96,10 @@ class PromptAlignmentMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -101,6 +108,7 @@ class PromptAlignmentMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -128,7 +136,10 @@ class PromptAlignmentMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str, actual_output: str) -> str:

deepeval/metrics/role_adherence/role_adherence.py

@@ -1,6 +1,7 @@
 from typing import Optional, Union, List
 
 from deepeval.metrics import BaseConversationalMetric
+from deepeval.metrics.api import metric_data_manager
 from deepeval.metrics.role_adherence.schema import (
     OutOfCharacterResponseVerdicts,
 )
@@ -44,6 +45,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case,
@@ -63,6 +65,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -82,6 +85,10 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -89,6 +96,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case,
@@ -124,6 +132,10 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, role: str) -> str:

deepeval/metrics/role_violation/role_violation.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.role_violation.template import RoleViolationTemplate
 from deepeval.metrics.role_violation.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class RoleViolationMetric(BaseMetric):
@@ -58,6 +59,7 @@ class RoleViolationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -73,6 +75,7 @@ class RoleViolationMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -94,6 +97,10 @@ class RoleViolationMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -102,6 +109,7 @@ class RoleViolationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -131,6 +139,10 @@ class RoleViolationMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/summarization/summarization.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Union
 import asyncio
 
+from deepeval.metrics.api import metric_data_manager
 from deepeval.test_case import (
     LLMTestCase,
     LLMTestCaseParams,
@@ -73,6 +74,7 @@ class SummarizationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -88,6 +90,7 @@ class SummarizationMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -121,7 +124,10 @@ class SummarizationMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -129,6 +135,7 @@ class SummarizationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -171,6 +178,10 @@ class SummarizationMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/task_completion/task_completion.py

@@ -50,6 +50,7 @@ class TaskCompletionMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         has_trace: bool = isinstance(test_case._trace_dict, Dict)
         if not has_trace:
@@ -66,6 +67,7 @@ class TaskCompletionMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +91,7 @@ class TaskCompletionMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         has_trace: bool = isinstance(test_case._trace_dict, Dict)
         if not has_trace:

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -12,6 +12,7 @@ from deepeval.test_case import (
     ToolCall,
 )
 from deepeval.metrics import BaseMetric
+from deepeval.metrics.api import metric_data_manager
 
 
 class ToolCorrectnessMetric(BaseMetric):
@@ -45,6 +46,7 @@ class ToolCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -83,6 +85,11 @@ class ToolCorrectnessMetric(BaseMetric):
             ]
             steps.append(f"Score: {self.score}\nReason: {self.reason}")
             self.verbose_logs = construct_verbose_logs(self, steps=steps)
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def a_measure(
@@ -90,6 +97,7 @@ class ToolCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         return self.measure(
             test_case,

deepeval/metrics/toxicity/toxicity.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.toxicity.template import ToxicityTemplate
 from deepeval.metrics.toxicity.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ToxicityMetric(BaseMetric):
@@ -50,6 +51,7 @@ class ToxicityMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -65,6 +67,7 @@ class ToxicityMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class ToxicityMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -92,6 +99,7 @@ class ToxicityMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -122,6 +130,10 @@ class ToxicityMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/turn_relevancy/turn_relevancy.py

@@ -20,6 +20,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, Turn, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.turn_relevancy.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class TurnRelevancyMetric(BaseConversationalMetric):
@@ -49,6 +50,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -65,6 +67,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -91,6 +94,10 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -98,6 +105,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -134,6 +142,10 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/models/llms/grok_model.py

@@ -56,8 +56,8 @@ model_pricing = {
 class GrokModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        api_key: Optional[str] = None,
         model: Optional[str] = None,
+        api_key: Optional[str] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,

deepeval/models/llms/openai_model.py

@@ -70,6 +70,8 @@ unsupported_log_probs_gpt_models = [
     "o1-mini-2024-09-12",
     "o3-mini",
     "o3-mini-2025-01-31",
+    "o4-mini",
+    "o4-mini-2025-04-16",
     "gpt-4.5-preview-2025-02-27",
     "gpt-5",
     "gpt-5-2025-08-07",

deepeval/openai/__init__.py

@@ -1,37 +1,19 @@
-from importlib.machinery import SourceFileLoader
-import importlib.util
-import sys
-
-from deepeval.openai.patch import patch_openai
+try:
+    import openai  # noqa: F401
+except ImportError:
+    raise ModuleNotFoundError(
+        "Please install OpenAI to use this feature: 'pip install openai'"
+    )
 
 
-def load_and_patch_openai():
-    openai_spec = importlib.util.find_spec("openai")
-    if not openai_spec or not openai_spec.origin:
-        raise ImportError("Could not find the OpenAI package")
-    package_dirs = openai_spec.submodule_search_locations
-    loader = SourceFileLoader("deepeval_openai", openai_spec.origin)
-    new_spec = importlib.util.spec_from_loader(
-        "deepeval_openai",
-        loader,
-        origin=openai_spec.origin,
-        is_package=True,
-    )
-    deepeval_openai = importlib.util.module_from_spec(new_spec)
-    deepeval_openai.__path__ = package_dirs
-    sys.modules["deepeval_openai"] = deepeval_openai
-    loader.exec_module(deepeval_openai)
-    patch_openai(deepeval_openai)
-    return deepeval_openai
+try:
+    from openai import OpenAI, AsyncOpenAI  # noqa: F401
+except ImportError:
+    OpenAI = None  # type: ignore
+    AsyncOpenAI = None  # type: ignore
 
 
-patched_openai = load_and_patch_openai()
-openai = patched_openai
-OpenAI = patched_openai.OpenAI
-AsyncOpenAI = patched_openai.AsyncOpenAI
+if OpenAI or AsyncOpenAI:
+    from deepeval.openai.patch import patch_openai_classes
 
-__all__ = [
-    "openai",
-    "OpenAI",
-    "AsyncOpenAI",
-]
+    patch_openai_classes()