deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/turn_relevancy/turn_relevancy.py

@@ -11,15 +11,19 @@ from deepeval.metrics.utils import (
     construct_verbose_logs,
     get_turns_in_sliding_window,
     get_unit_interactions,
-    trimAndLoadJson,
     initialize_model,
     convert_turn_to_dict,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, Turn, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.turn_relevancy.schema import *
+from deepeval.metrics.turn_relevancy.schema import (
+    TurnRelevancyVerdict,
+    TurnRelevancyScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -53,7 +57,12 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -108,7 +117,12 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -148,7 +162,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -162,24 +176,19 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         prompt = TurnRelevancyTemplate.generate_reason(
             score=self.score, irrelevancies=irrelevancies
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TurnRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: TurnRelevancyScoreReason = await self.model.a_generate(
-                    prompt, schema=TurnRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
-    def _generate_reason(self) -> str:
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
+
+    def _generate_reason(self) -> Optional[str]:
+        if self.include_reason is False:
+            return None
+
         irrelevancies: List[Dict[str, str]] = []
         for index, verdict in enumerate(self.verdicts):
             if verdict.verdict.strip().lower() == "no":
@@ -190,22 +199,14 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         prompt = TurnRelevancyTemplate.generate_reason(
             score=self.score, irrelevancies=irrelevancies
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=TurnRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: TurnRelevancyScoreReason = self.model.generate(
-                    prompt, schema=TurnRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdict(
         self, turns_sliding_window: List[Turn]
@@ -215,22 +216,14 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                 convert_turn_to_dict(turn) for turn in turns_sliding_window
             ]
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TurnRelevancyVerdict
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: TurnRelevancyVerdict = await self.model.a_generate(
-                    prompt, schema=TurnRelevancyVerdict
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TurnRelevancyVerdict(**data)
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TurnRelevancyVerdict(**data),
+        )
 
     def _generate_verdict(
         self, turns_sliding_window: List[Turn]
@@ -240,20 +233,14 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                 convert_turn_to_dict(turn) for turn in turns_sliding_window
             ]
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=TurnRelevancyVerdict)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: TurnRelevancyVerdict = self.model.generate(
-                    prompt, schema=TurnRelevancyVerdict
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TurnRelevancyVerdict(**data)
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TurnRelevancyVerdict(**data),
+        )
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -274,7 +261,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         else:
             try:
                 self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/utils.py

@@ -2,16 +2,24 @@ import inspect
 import json
 import re
 import sys
-import itertools
-from typing import Any, Dict, Optional, List, Union, Tuple
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
 
 from deepeval.errors import (
     MissingTestCaseParamsError,
-    MismatchedTestCaseInputsError,
 )
+from deepeval.utils import convert_to_multi_modal_array
 from deepeval.models import (
     DeepEvalBaseLLM,
-    DeepEvalBaseMLLM,
     GPTModel,
     AnthropicModel,
     AzureOpenAIModel,
@@ -22,16 +30,20 @@ from deepeval.models import (
     OllamaEmbeddingModel,
     LocalEmbeddingModel,
     GeminiModel,
-    MultimodalOpenAIModel,
-    MultimodalGeminiModel,
-    MultimodalOllamaModel,
-    MultimodalAzureOpenAIMLLMModel,
     AmazonBedrockModel,
     LiteLLMModel,
     KimiModel,
     GrokModel,
     DeepSeekModel,
 )
+from deepeval.models.llms.constants import (
+    OPENAI_MODELS_DATA,
+    GEMINI_MODELS_DATA,
+    OLLAMA_MODELS_DATA,
+    ANTHROPIC_MODELS_DATA,
+    GROK_MODELS_DATA,
+    KIMI_MODELS_DATA,
+)
 from deepeval.key_handler import (
     ModelKeyValues,
     EmbeddingKeyValues,
@@ -40,16 +52,12 @@ from deepeval.key_handler import (
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
     BaseArenaMetric,
 )
 from deepeval.models.base_model import DeepEvalBaseEmbeddingModel
 from deepeval.test_case import (
-    Turn,
     LLMTestCase,
     LLMTestCaseParams,
-    MLLMTestCase,
-    MLLMTestCaseParams,
     ConversationalTestCase,
     MLLMImage,
     Turn,
@@ -58,12 +66,20 @@ from deepeval.test_case import (
     TurnParams,
 )
 
+MULTIMODAL_SUPPORTED_MODELS = {
+    GPTModel: OPENAI_MODELS_DATA,
+    GeminiModel: GEMINI_MODELS_DATA,
+    OllamaModel: OLLAMA_MODELS_DATA,
+    AzureOpenAIModel: OPENAI_MODELS_DATA,
+    KimiModel: KIMI_MODELS_DATA,
+    AnthropicModel: ANTHROPIC_MODELS_DATA,
+    GrokModel: GROK_MODELS_DATA,
+}
+
 
 def copy_metrics(
-    metrics: List[
-        Union[BaseMetric, BaseConversationalMetric, BaseMultimodalMetric]
-    ],
-) -> List[Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric]]:
+    metrics: List[Union[BaseMetric, BaseConversationalMetric]],
+) -> List[Union[BaseMetric, BaseConversationalMetric]]:
     copied_metrics = []
     for metric in metrics:
         metric_class = type(metric)
@@ -201,7 +217,28 @@ def check_conversational_test_case_params(
     test_case_params: List[TurnParams],
     metric: BaseConversationalMetric,
     require_chatbot_role: bool = False,
+    model: Optional[DeepEvalBaseLLM] = None,
+    multimodal: Optional[bool] = False,
 ):
+    if multimodal:
+        if not model or not model.supports_multimodal():
+            if model and type(model) in MULTIMODAL_SUPPORTED_MODELS.keys():
+                valid_multimodal_models = []
+                for model_name, model_data in MULTIMODAL_SUPPORTED_MODELS.get(
+                    type(model)
+                ).items():
+                    if callable(model_data):
+                        model_data = model_data()
+                    if model_data.supports_multimodal:
+                        valid_multimodal_models.append(model_name)
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(valid_multimodal_models)}."
+                )
+            else:
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS.keys()])}"
+                )
+
     if isinstance(test_case, ConversationalTestCase) is False:
         error_str = f"Unable to evaluate test cases that are not of type 'ConversationalTestCase' using the conversational '{metric.__name__}' metric."
         metric.error = error_str
@@ -234,8 +271,49 @@ def check_conversational_test_case_params(
 def check_llm_test_case_params(
     test_case: LLMTestCase,
     test_case_params: List[LLMTestCaseParams],
+    input_image_count: Optional[int],
+    actual_output_image_count: Optional[int],
     metric: Union[BaseMetric, BaseArenaMetric],
+    model: Optional[DeepEvalBaseLLM] = None,
+    multimodal: Optional[bool] = False,
 ):
+    if multimodal:
+        if not model or not model.supports_multimodal():
+            if model and type(model) in MULTIMODAL_SUPPORTED_MODELS.keys():
+                valid_multimodal_models = []
+                for model_name, model_data in MULTIMODAL_SUPPORTED_MODELS.get(
+                    type(model)
+                ).items():
+                    if callable(model_data):
+                        model_data = model_data()
+                    if model_data.supports_multimodal:
+                        valid_multimodal_models.append(model_name)
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(valid_multimodal_models)}."
+                )
+            else:
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS.keys()])}"
+                )
+
+        if input_image_count:
+            count = 0
+            for ele in convert_to_multi_modal_array(test_case.input):
+                if isinstance(ele, MLLMImage):
+                    count += 1
+            if count != input_image_count:
+                error_str = f"Can only evaluate test cases with '{input_image_count}' input images using the '{metric.__name__}' metric. `{count}` found."
+                raise ValueError(error_str)
+
+        if actual_output_image_count:
+            count = 0
+            for ele in convert_to_multi_modal_array(test_case.actual_output):
+                if isinstance(ele, MLLMImage):
+                    count += 1
+            if count != actual_output_image_count:
+                error_str = f"Unable to evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
+                raise ValueError(error_str)
+
     if isinstance(test_case, LLMTestCase) is False:
         error_str = f"Unable to evaluate test cases that are not of type 'LLMTestCase' using the non-conversational '{metric.__name__}' metric."
         metric.error = error_str
@@ -265,6 +343,8 @@ def check_arena_test_case_params(
     arena_test_case: ArenaTestCase,
     test_case_params: List[LLMTestCaseParams],
     metric: BaseArenaMetric,
+    model: Optional[DeepEvalBaseLLM] = None,
+    multimodal: Optional[bool] = False,
 ):
     if not isinstance(arena_test_case, ArenaTestCase):
         raise ValueError(
@@ -285,73 +365,8 @@ def check_arena_test_case_params(
             )
 
     for test_case in cases:
-        check_llm_test_case_params(test_case, test_case_params, metric)
-
-
-def check_mllm_test_case_params(
-    test_case: MLLMTestCase,
-    test_case_params: List[MLLMTestCaseParams],
-    input_image_count: Optional[int],
-    actual_output_image_count: Optional[int],
-    metric: BaseMetric,
-):
-    if input_image_count:
-        count = 0
-        for ele in test_case.input:
-            if isinstance(ele, MLLMImage):
-                count += 1
-        if count != input_image_count:
-            error_str = f"Can only evaluate test cases with '{input_image_count}' input images using the '{metric.__name__}' metric. `{count}` found."
-            raise ValueError(error_str)
-
-    if actual_output_image_count:
-        count = 0
-        for ele in test_case.actual_output:
-            if isinstance(ele, MLLMImage):
-                count += 1
-        if count != actual_output_image_count:
-            error_str = f"Unable to evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
-            raise ValueError(error_str)
-
-    if isinstance(test_case, MLLMTestCase) is False:
-        error_str = f"Unable to evaluate test cases that are not of type 'MLLMTestCase' using the '{metric.__name__}' metric."
-        metric.error = error_str
-        raise ValueError(error_str)
-
-    missing_params = []
-    for param in test_case_params:
-        if getattr(test_case, param.value) is None:
-            missing_params.append(f"'{param.value}'")
-
-    if missing_params:
-        if len(missing_params) == 1:
-            missing_params_str = missing_params[0]
-        elif len(missing_params) == 2:
-            missing_params_str = " and ".join(missing_params)
-        else:
-            missing_params_str = (
-                ", ".join(missing_params[:-1]) + ", and " + missing_params[-1]
-            )
-
-        error_str = f"{missing_params_str} cannot be None for the '{metric.__name__}' metric"
-        metric.error = error_str
-        raise MissingTestCaseParamsError(error_str)
-
-
-def check_mllm_test_cases_params(
-    test_cases: List[MLLMTestCase],
-    test_case_params: List[MLLMTestCaseParams],
-    input_image_count: Optional[int],
-    actual_output_image_count: Optional[int],
-    metric: BaseMetric,
-):
-    for test_case in test_cases:
-        check_mllm_test_case_params(
-            test_case,
-            test_case_params,
-            input_image_count,
-            actual_output_image_count,
-            metric,
+        check_llm_test_case_params(
+            test_case, test_case_params, None, None, metric, model, multimodal
         )
 
 
@@ -381,6 +396,63 @@ def trimAndLoadJson(
         raise Exception(f"An unexpected error occurred: {str(e)}")
 
 
+SchemaType = TypeVar("SchemaType")
+ReturnType = TypeVar("ReturnType")
+
+
+def generate_with_schema_and_extract(
+    metric: Union[BaseMetric, BaseArenaMetric, BaseConversationalMetric],
+    prompt: Any,
+    schema_cls: Type[SchemaType],
+    *,
+    extract_schema: Callable[[SchemaType], ReturnType],
+    extract_json: Callable[[Dict[str, Any]], ReturnType],
+) -> ReturnType:
+    """
+    Synchronous wrapper:
+    - calls model.generate_with_schema(...)
+    - accrues cost if applicable
+    - if schema instance -> extract_schema
+      else parse JSON -> extract_json
+    """
+    if metric.using_native_model:
+        result, cost = metric.model.generate_with_schema(
+            prompt, schema=schema_cls
+        )
+        metric._accrue_cost(cost)
+    else:
+        result = metric.model.generate_with_schema(prompt, schema=schema_cls)
+    if isinstance(result, schema_cls):
+        return extract_schema(result)
+    data = trimAndLoadJson(result, metric)
+    return extract_json(data)
+
+
+async def a_generate_with_schema_and_extract(
+    metric: Union[BaseMetric, BaseArenaMetric, BaseConversationalMetric],
+    prompt: Any,
+    schema_cls: Type[SchemaType],
+    *,
+    extract_schema: Callable[[SchemaType], ReturnType],
+    extract_json: Callable[[Dict[str, Any]], ReturnType],
+) -> ReturnType:
+    if metric.using_native_model:
+        result, cost = await metric.model.a_generate_with_schema(
+            prompt, schema=schema_cls
+        )
+        metric._accrue_cost(cost)
+    else:
+        result = await metric.model.a_generate_with_schema(
+            prompt, schema=schema_cls
+        )
+
+    if isinstance(result, schema_cls):
+        return extract_schema(result)
+
+    data = trimAndLoadJson(result, metric)
+    return extract_json(data)
+
+
 ###############################################
 # Default Model Providers
 ###############################################
@@ -397,8 +469,8 @@ def should_use_local_model():
 
 
 def should_use_ollama_model():
-    base_url = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_API_KEY)
-    return base_url == "ollama"
+    value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_API_KEY)
+    return value == "ollama"
 
 
 def should_use_gemini_model():
@@ -459,7 +531,7 @@ def initialize_model(
     elif should_use_local_model():
         return LocalModel(), True
     elif should_use_azure_openai():
-        return AzureOpenAIModel(model_name=model), True
+        return AzureOpenAIModel(model=model), True
     elif should_use_moonshot_model():
         return KimiModel(model=model), True
     elif should_use_grok_model():
@@ -501,42 +573,6 @@ def is_native_model(
 ###############################################
 
 
-def initialize_multimodal_model(
-    model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
-) -> Tuple[DeepEvalBaseLLM, bool]:
-    """
-    Returns a tuple of (initialized DeepEvalBaseMLLM, using_native_model boolean)
-    """
-    if is_native_mllm(model):
-        return model, True
-    if isinstance(model, DeepEvalBaseMLLM):
-        return model, False
-    if should_use_gemini_model():
-        return MultimodalGeminiModel(), True
-    if should_use_ollama_model():
-        return MultimodalOllamaModel(), True
-    elif should_use_azure_openai():
-        return MultimodalAzureOpenAIMLLMModel(model_name=model), True
-    elif isinstance(model, str) or model is None:
-        return MultimodalOpenAIModel(model=model), True
-    raise TypeError(
-        f"Unsupported type for model: {type(model)}. Expected None, str, DeepEvalBaseMLLM, MultimodalOpenAIModel, MultimodalOllamaModel."
-    )
-
-
-def is_native_mllm(
-    model: Optional[Union[str, DeepEvalBaseLLM]] = None,
-) -> bool:
-    if (
-        isinstance(model, MultimodalOpenAIModel)
-        or isinstance(model, MultimodalOllamaModel)
-        or isinstance(model, MultimodalGeminiModel)
-    ):
-        return True
-    else:
-        return False
-
-
 ###############################################
 # Embedding Model
 ###############################################

deepeval/models/__init__.py

@@ -1,7 +1,6 @@
 from deepeval.models.base_model import (
     DeepEvalBaseModel,
     DeepEvalBaseLLM,
-    DeepEvalBaseMLLM,
     DeepEvalBaseEmbeddingModel,
 )
 from deepeval.models.llms import (
@@ -17,12 +16,6 @@ from deepeval.models.llms import (
     GrokModel,
     DeepSeekModel,
 )
-from deepeval.models.mlllms import (
-    MultimodalOpenAIModel,
-    MultimodalOllamaModel,
-    MultimodalGeminiModel,
-    MultimodalAzureOpenAIMLLMModel,
-)
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
     AzureOpenAIEmbeddingModel,
@@ -33,7 +26,6 @@ from deepeval.models.embedding_models import (
 __all__ = [
     "DeepEvalBaseModel",
     "DeepEvalBaseLLM",
-    "DeepEvalBaseMLLM",
     "DeepEvalBaseEmbeddingModel",
     "GPTModel",
     "AzureOpenAIModel",
@@ -46,10 +38,6 @@ __all__ = [
     "KimiModel",
     "GrokModel",
     "DeepSeekModel",
-    "MultimodalOpenAIModel",
-    "MultimodalOllamaModel",
-    "MultimodalGeminiModel",
-    "MultimodalAzureOpenAIMLLMModel",
     "OpenAIEmbeddingModel",
     "AzureOpenAIEmbeddingModel",
     "LocalEmbeddingModel",