deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/utils.py

@@ -2,16 +2,14 @@ import inspect
 import json
 import re
 import sys
-import itertools
 from typing import Any, Dict, Optional, List, Union, Tuple
 
 from deepeval.errors import (
     MissingTestCaseParamsError,
-    MismatchedTestCaseInputsError,
 )
+from deepeval.utils import convert_to_multi_modal_array
 from deepeval.models import (
     DeepEvalBaseLLM,
-    DeepEvalBaseMLLM,
     GPTModel,
     AnthropicModel,
     AzureOpenAIModel,
@@ -22,10 +20,6 @@ from deepeval.models import (
     OllamaEmbeddingModel,
     LocalEmbeddingModel,
     GeminiModel,
-    MultimodalOpenAIModel,
-    MultimodalGeminiModel,
-    MultimodalOllamaModel,
-    MultimodalAzureOpenAIMLLMModel,
     AmazonBedrockModel,
     LiteLLMModel,
     KimiModel,
@@ -45,11 +39,8 @@ from deepeval.metrics import (
 )
 from deepeval.models.base_model import DeepEvalBaseEmbeddingModel
 from deepeval.test_case import (
-    Turn,
     LLMTestCase,
     LLMTestCaseParams,
-    MLLMTestCase,
-    MLLMTestCaseParams,
     ConversationalTestCase,
     MLLMImage,
     Turn,
@@ -58,6 +49,13 @@ from deepeval.test_case import (
     TurnParams,
 )
 
+MULTIMODAL_SUPPORTED_MODELS = [
+    GPTModel,
+    GeminiModel,
+    OllamaModel,
+    AzureOpenAIModel,
+]
+
 
 def copy_metrics(
     metrics: List[
@@ -201,7 +199,20 @@ def check_conversational_test_case_params(
     test_case_params: List[TurnParams],
     metric: BaseConversationalMetric,
     require_chatbot_role: bool = False,
+    model: Optional[DeepEvalBaseLLM] = None,
+    multimodal: Optional[bool] = False,
 ):
+    if multimodal:
+        if not model or not model.supports_multimodal():
+            if model and type(model) in MULTIMODAL_SUPPORTED_MODELS:
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(model.__class__.valid_multimodal_models)}."
+                )
+            else:
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+                )
+
     if isinstance(test_case, ConversationalTestCase) is False:
         error_str = f"Unable to evaluate test cases that are not of type 'ConversationalTestCase' using the conversational '{metric.__name__}' metric."
         metric.error = error_str
@@ -289,15 +300,26 @@ def check_arena_test_case_params(
 
 
 def check_mllm_test_case_params(
-    test_case: MLLMTestCase,
-    test_case_params: List[MLLMTestCaseParams],
+    test_case: LLMTestCase,
+    test_case_params: List[LLMTestCaseParams],
     input_image_count: Optional[int],
     actual_output_image_count: Optional[int],
     metric: BaseMetric,
+    model: Optional[DeepEvalBaseLLM] = None,
 ):
+    if not model or not model.supports_multimodal():
+        if model and type(model) in MULTIMODAL_SUPPORTED_MODELS:
+            raise ValueError(
+                f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(model.__class__.valid_multimodal_models)}."
+            )
+        else:
+            raise ValueError(
+                f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+            )
+
     if input_image_count:
         count = 0
-        for ele in test_case.input:
+        for ele in convert_to_multi_modal_array(test_case.input):
             if isinstance(ele, MLLMImage):
                 count += 1
         if count != input_image_count:
@@ -306,18 +328,13 @@ def check_mllm_test_case_params(
 
     if actual_output_image_count:
         count = 0
-        for ele in test_case.actual_output:
+        for ele in convert_to_multi_modal_array(test_case.actual_output):
             if isinstance(ele, MLLMImage):
                 count += 1
         if count != actual_output_image_count:
             error_str = f"Unable to evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
             raise ValueError(error_str)
 
-    if isinstance(test_case, MLLMTestCase) is False:
-        error_str = f"Unable to evaluate test cases that are not of type 'MLLMTestCase' using the '{metric.__name__}' metric."
-        metric.error = error_str
-        raise ValueError(error_str)
-
     missing_params = []
     for param in test_case_params:
         if getattr(test_case, param.value) is None:
@@ -339,8 +356,8 @@ def check_mllm_test_case_params(
 
 
 def check_mllm_test_cases_params(
-    test_cases: List[MLLMTestCase],
-    test_case_params: List[MLLMTestCaseParams],
+    test_cases: List[LLMTestCase],
+    test_case_params: List[LLMTestCaseParams],
     input_image_count: Optional[int],
     actual_output_image_count: Optional[int],
     metric: BaseMetric,
@@ -459,7 +476,7 @@ def initialize_model(
     elif should_use_local_model():
         return LocalModel(), True
     elif should_use_azure_openai():
-        return AzureOpenAIModel(model_name=model), True
+        return AzureOpenAIModel(model=model), True
     elif should_use_moonshot_model():
         return KimiModel(model=model), True
     elif should_use_grok_model():
@@ -501,42 +518,6 @@ def is_native_model(
 ###############################################
 
 
-def initialize_multimodal_model(
-    model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
-) -> Tuple[DeepEvalBaseLLM, bool]:
-    """
-    Returns a tuple of (initialized DeepEvalBaseMLLM, using_native_model boolean)
-    """
-    if is_native_mllm(model):
-        return model, True
-    if isinstance(model, DeepEvalBaseMLLM):
-        return model, False
-    if should_use_gemini_model():
-        return MultimodalGeminiModel(), True
-    if should_use_ollama_model():
-        return MultimodalOllamaModel(), True
-    elif should_use_azure_openai():
-        return MultimodalAzureOpenAIMLLMModel(model_name=model), True
-    elif isinstance(model, str) or model is None:
-        return MultimodalOpenAIModel(model=model), True
-    raise TypeError(
-        f"Unsupported type for model: {type(model)}. Expected None, str, DeepEvalBaseMLLM, MultimodalOpenAIModel, MultimodalOllamaModel."
-    )
-
-
-def is_native_mllm(
-    model: Optional[Union[str, DeepEvalBaseLLM]] = None,
-) -> bool:
-    if (
-        isinstance(model, MultimodalOpenAIModel)
-        or isinstance(model, MultimodalOllamaModel)
-        or isinstance(model, MultimodalGeminiModel)
-    ):
-        return True
-    else:
-        return False
-
-
 ###############################################
 # Embedding Model
 ###############################################

deepeval/models/__init__.py

@@ -1,7 +1,6 @@
 from deepeval.models.base_model import (
     DeepEvalBaseModel,
     DeepEvalBaseLLM,
-    DeepEvalBaseMLLM,
     DeepEvalBaseEmbeddingModel,
 )
 from deepeval.models.llms import (
@@ -17,12 +16,6 @@ from deepeval.models.llms import (
     GrokModel,
     DeepSeekModel,
 )
-from deepeval.models.mlllms import (
-    MultimodalOpenAIModel,
-    MultimodalOllamaModel,
-    MultimodalGeminiModel,
-    MultimodalAzureOpenAIMLLMModel,
-)
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
     AzureOpenAIEmbeddingModel,
@@ -33,7 +26,6 @@ from deepeval.models.embedding_models import (
 __all__ = [
     "DeepEvalBaseModel",
     "DeepEvalBaseLLM",
-    "DeepEvalBaseMLLM",
     "DeepEvalBaseEmbeddingModel",
     "GPTModel",
     "AzureOpenAIModel",
@@ -46,10 +38,6 @@ __all__ = [
     "KimiModel",
     "GrokModel",
     "DeepSeekModel",
-    "MultimodalOpenAIModel",
-    "MultimodalOllamaModel",
-    "MultimodalGeminiModel",
-    "MultimodalAzureOpenAIMLLMModel",
     "OpenAIEmbeddingModel",
     "AzureOpenAIEmbeddingModel",
     "LocalEmbeddingModel",

deepeval/models/base_model.py

@@ -31,9 +31,9 @@ class DeepEvalBaseModel(ABC):
 
 
 class DeepEvalBaseLLM(ABC):
-    def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
-        self.model_name = parse_model_name(model_name)
-        self.model = self.load_model(*args, **kwargs)
+    def __init__(self, model: Optional[str] = None, *args, **kwargs):
+        self.name = parse_model_name(model)
+        self.model = self.load_model()
 
     @abstractmethod
     def load_model(self, *args, **kwargs) -> "DeepEvalBaseLLM":
@@ -62,6 +62,13 @@ class DeepEvalBaseLLM(ABC):
         """
         pass
 
+    @abstractmethod
+    def get_model_name(self, *args, **kwargs) -> str:
+        return self.name
+
+    def supports_multimodal(self) -> bool:
+        return False
+
     def batch_generate(self, *args, **kwargs) -> List[str]:
         """Runs the model to output LLM responses.
 
@@ -72,43 +79,14 @@ class DeepEvalBaseLLM(ABC):
             "batch_generate is not implemented for this model"
         )
 
-    @abstractmethod
-    def get_model_name(self, *args, **kwargs) -> str:
-        pass
-
-
-class DeepEvalBaseMLLM(ABC):
-    def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
-        self.model_name = parse_model_name(model_name)
-
-    @abstractmethod
-    def generate(self, *args, **kwargs) -> str:
-        """Runs the model to output MLLM response.
-
-        Returns:
-            A string.
-        """
-        pass
-
-    @abstractmethod
-    async def a_generate(self, *args, **kwargs) -> str:
-        """Runs the model to output MLLM response.
-
-        Returns:
-            A string.
-        """
-        pass
-
-    @abstractmethod
-    def get_model_name(self, *args, **kwargs) -> str:
-        pass
+    def supports_multimodal(self):
+        return False
 
 
 class DeepEvalBaseEmbeddingModel(ABC):
-    def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
-        self.model_name = parse_model_name(model_name)
-
-        self.model = self.load_model(*args, **kwargs)
+    def __init__(self, model: Optional[str] = None, *args, **kwargs):
+        self.name = parse_model_name(model)
+        self.model = self.load_model()
 
     @abstractmethod
     def load_model(self, *args, **kwargs) -> "DeepEvalBaseEmbeddingModel":
@@ -157,4 +135,4 @@ class DeepEvalBaseEmbeddingModel(ABC):
 
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str:
-        pass
+        return self.name

deepeval/models/embedding_models/__init__.py

@@ -2,3 +2,10 @@ from .azure_embedding_model import AzureOpenAIEmbeddingModel
 from .openai_embedding_model import OpenAIEmbeddingModel
 from .local_embedding_model import LocalEmbeddingModel
 from .ollama_embedding_model import OllamaEmbeddingModel
+
+__all__ = [
+    "AzureOpenAIEmbeddingModel",
+    "OpenAIEmbeddingModel",
+    "LocalEmbeddingModel",
+    "OllamaEmbeddingModel",
+]

deepeval/models/embedding_models/azure_embedding_model.py

@@ -1,55 +1,85 @@
 from typing import Dict, List, Optional
 from openai import AzureOpenAI, AsyncAzureOpenAI
-from deepeval.key_handler import (
-    EmbeddingKeyValues,
-    ModelKeyValues,
-    KEY_FILE_HANDLER,
-)
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
 from deepeval.models import DeepEvalBaseEmbeddingModel
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.constants import ProviderSlug as PS
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
 
 
 retry_azure = create_retry_decorator(PS.AZURE)
 
+_ALIAS_MAP = {
+    "api_key": ["openai_api_key"],
+    "base_url": ["azure_endpoint"],
+    "deployment_name": ["azure_deployment"],
+}
+
 
 class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def __init__(
         self,
-        openai_api_key: Optional[str] = None,
-        openai_api_version: Optional[str] = None,
-        azure_endpoint: Optional[str] = None,
-        azure_deployment: Optional[str] = None,
         model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        deployment_name: Optional[str] = None,
+        openai_api_version: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
-        self.openai_api_key = openai_api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.AZURE_OPENAI_API_KEY
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "AzureOpenAIEmbeddingModel",
+            kwargs,
+            _ALIAS_MAP,
         )
+
+        # re-map depricated keywords to re-named positional args
+        if api_key is None and "api_key" in alias_values:
+            api_key = alias_values["api_key"]
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+        if deployment_name is None and "deployment_name" in alias_values:
+            deployment_name = alias_values["deployment_name"]
+
+        settings = get_settings()
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.AZURE_OPENAI_API_KEY
+
         self.openai_api_version = (
-            openai_api_version
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.OPENAI_API_VERSION)
+            openai_api_version or settings.OPENAI_API_VERSION
         )
-        self.azure_endpoint = azure_endpoint or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.AZURE_OPENAI_ENDPOINT
+        self.base_url = (
+            base_url
+            or settings.AZURE_OPENAI_ENDPOINT
+            and str(settings.AZURE_OPENAI_ENDPOINT)
         )
-        self.azure_deployment = azure_deployment or KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.AZURE_EMBEDDING_DEPLOYMENT_NAME
+
+        self.deployment_name = (
+            deployment_name or settings.AZURE_EMBEDDING_DEPLOYMENT_NAME
         )
-        self.client_kwargs = client_kwargs or {}
-        self.model_name = model or self.azure_deployment
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
+        model = model or self.deployment_name
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(self.model_name)
+        super().__init__(model)
 
     @retry_azure
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=text, model=self.model_name, **self.generation_kwargs
+            input=text, model=self.name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -57,7 +87,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=texts, model=self.model_name, **self.generation_kwargs
+            input=texts, model=self.name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -65,7 +95,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=text, model=self.model_name, **self.generation_kwargs
+            input=text, model=self.name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -73,28 +103,32 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=texts, model=self.model_name, **self.generation_kwargs
+            input=texts, model=self.name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
-    def get_model_name(self) -> str:
-        return self.model_name
-
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(AzureOpenAI)
         return self._build_client(AsyncAzureOpenAI)
 
     def _build_client(self, cls):
-        client_kwargs = self.client_kwargs.copy()
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="AzureOpenAI",
+            env_var_name="AZURE_OPENAI_API_KEY",
+            param_hint="`api_key` to AzureOpenAIEmbeddingModel(...)",
+        )
+
+        client_kwargs = self.kwargs.copy()
         if not sdk_retries_for(PS.AZURE):
             client_kwargs["max_retries"] = 0
 
         client_init_kwargs = dict(
-            api_key=self.openai_api_key,
+            api_key=api_key,
             api_version=self.openai_api_version,
-            azure_endpoint=self.azure_endpoint,
-            azure_deployment=self.azure_deployment,
+            azure_endpoint=self.base_url,
+            azure_deployment=self.deployment_name,
             **client_kwargs,
         )
         try:
@@ -105,3 +139,6 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
                 client_init_kwargs.pop("max_retries", None)
                 return cls(**client_init_kwargs)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Azure)"

deepeval/models/embedding_models/local_embedding_model.py

@@ -1,7 +1,11 @@
 from openai import OpenAI, AsyncOpenAI
 from typing import Dict, List, Optional
+from pydantic import SecretStr
 
-from deepeval.key_handler import EmbeddingKeyValues, KEY_FILE_HANDLER
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseEmbeddingModel
 from deepeval.models.retry_policy import (
     create_retry_decorator,
@@ -17,30 +21,36 @@ retry_local = create_retry_decorator(PS.LOCAL)
 class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def __init__(
         self,
+        model: Optional[str] = None,
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
-        model: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_API_KEY
-        )
-        self.base_url = base_url or KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_BASE_URL
-        )
-        self.model_name = model or KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_MODEL_NAME
+
+        settings = get_settings()
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = get_settings().LOCAL_EMBEDDING_API_KEY
+
+        self.base_url = (
+            base_url
+            or settings.LOCAL_EMBEDDING_BASE_URL
+            and str(settings.LOCAL_EMBEDDING_BASE_URL)
         )
-        self.client_kwargs = client_kwargs or {}
+        model = model or settings.LOCAL_EMBEDDING_MODEL_NAME
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(self.model_name)
+        super().__init__(model)
 
     @retry_local
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
-            model=self.model_name, input=[text], **self.generation_kwargs
+            model=self.name, input=[text], **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -48,7 +58,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return [data.embedding for data in response.data]
 
@@ -56,7 +66,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
-            model=self.model_name, input=[text], **self.generation_kwargs
+            model=self.name, input=[text], **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -64,7 +74,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return [data.embedding for data in response.data]
 
@@ -72,21 +82,25 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     # Model
     ###############################################
 
-    def get_model_name(self):
-        return self.model_name
-
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(OpenAI)
         return self._build_client(AsyncOpenAI)
 
     def _build_client(self, cls):
-        client_kwargs = self.client_kwargs.copy()
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="OpenAI",
+            env_var_name="LOCAL_EMBEDDING_API_KEY",
+            param_hint="`api_key` to LocalEmbeddingModel(...)",
+        )
+
+        client_kwargs = self.kwargs.copy()
         if not sdk_retries_for(PS.LOCAL):
             client_kwargs["max_retries"] = 0
 
         client_init_kwargs = dict(
-            api_key=self.api_key,
+            api_key=api_key,
             base_url=self.base_url,
             **client_kwargs,
         )
@@ -98,3 +112,6 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
                 client_init_kwargs.pop("max_retries", None)
                 return cls(**client_init_kwargs)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Local Model)"