deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/models/base_model.py

@@ -1,6 +1,18 @@
 from abc import ABC, abstractmethod
-from typing import Any, Optional, List
+from typing import Any, Optional, List, Union
 from deepeval.models.utils import parse_model_name
+from dataclasses import dataclass
+
+
+@dataclass
+class DeepEvalModelData:
+    supports_log_probs: Optional[bool] = None
+    supports_multimodal: Optional[bool] = None
+    supports_structured_outputs: Optional[bool] = None
+    supports_json: Optional[bool] = None
+    input_price: Optional[float] = None
+    output_price: Optional[float] = None
+    supports_temperature: Optional[bool] = True
 
 
 class DeepEvalBaseModel(ABC):
@@ -31,9 +43,9 @@ class DeepEvalBaseModel(ABC):
 
 
 class DeepEvalBaseLLM(ABC):
-    def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
-        self.model_name = parse_model_name(model_name)
-        self.model = self.load_model(*args, **kwargs)
+    def __init__(self, model: Optional[str] = None, *args, **kwargs):
+        self.name = parse_model_name(model)
+        self.model = self.load_model()
 
     @abstractmethod
     def load_model(self, *args, **kwargs) -> "DeepEvalBaseLLM":
@@ -62,6 +74,10 @@ class DeepEvalBaseLLM(ABC):
         """
         pass
 
+    @abstractmethod
+    def get_model_name(self, *args, **kwargs) -> str:
+        return self.name
+
     def batch_generate(self, *args, **kwargs) -> List[str]:
         """Runs the model to output LLM responses.
 
@@ -72,43 +88,43 @@ class DeepEvalBaseLLM(ABC):
             "batch_generate is not implemented for this model"
         )
 
-    @abstractmethod
-    def get_model_name(self, *args, **kwargs) -> str:
-        pass
+    # Capabilities
+    def supports_log_probs(self) -> Union[bool, None]:
+        return None
 
+    def supports_temperature(self) -> Union[bool, None]:
+        return None
 
-class DeepEvalBaseMLLM(ABC):
-    def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
-        self.model_name = parse_model_name(model_name)
+    def supports_multimodal(self) -> Union[bool, None]:
+        return None
 
-    @abstractmethod
-    def generate(self, *args, **kwargs) -> str:
-        """Runs the model to output MLLM response.
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return None
 
-        Returns:
-            A string.
-        """
-        pass
+    def supports_json_mode(self) -> Union[bool, None]:
+        return None
 
-    @abstractmethod
-    async def a_generate(self, *args, **kwargs) -> str:
-        """Runs the model to output MLLM response.
+    def generate_with_schema(self, *args, schema=None, **kwargs):
+        if schema is not None:
+            try:
+                return self.generate(*args, schema=schema, **kwargs)
+            except TypeError:
+                pass  # this means provider doesn't accept schema kwarg
+        return self.generate(*args, **kwargs)
 
-        Returns:
-            A string.
-        """
-        pass
-
-    @abstractmethod
-    def get_model_name(self, *args, **kwargs) -> str:
-        pass
+    async def a_generate_with_schema(self, *args, schema=None, **kwargs):
+        if schema is not None:
+            try:
+                return await self.a_generate(*args, schema=schema, **kwargs)
+            except TypeError:
+                pass
+        return await self.a_generate(*args, **kwargs)
 
 
 class DeepEvalBaseEmbeddingModel(ABC):
-    def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
-        self.model_name = parse_model_name(model_name)
-
-        self.model = self.load_model(*args, **kwargs)
+    def __init__(self, model: Optional[str] = None, *args, **kwargs):
+        self.name = parse_model_name(model)
+        self.model = self.load_model()
 
     @abstractmethod
     def load_model(self, *args, **kwargs) -> "DeepEvalBaseEmbeddingModel":
@@ -157,4 +173,4 @@ class DeepEvalBaseEmbeddingModel(ABC):
 
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str:
-        pass
+        return self.name

deepeval/models/embedding_models/__init__.py

@@ -2,3 +2,10 @@ from .azure_embedding_model import AzureOpenAIEmbeddingModel
 from .openai_embedding_model import OpenAIEmbeddingModel
 from .local_embedding_model import LocalEmbeddingModel
 from .ollama_embedding_model import OllamaEmbeddingModel
+
+__all__ = [
+    "AzureOpenAIEmbeddingModel",
+    "OpenAIEmbeddingModel",
+    "LocalEmbeddingModel",
+    "OllamaEmbeddingModel",
+]

deepeval/models/embedding_models/azure_embedding_model.py

@@ -9,53 +9,99 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.constants import ProviderSlug as PS
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
+from deepeval.utils import require_param
 
 
 retry_azure = create_retry_decorator(PS.AZURE)
 
+_ALIAS_MAP = {
+    "api_key": ["openai_api_key"],
+    "base_url": ["azure_endpoint"],
+    "deployment_name": ["azure_deployment"],
+}
+
 
 class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def __init__(
         self,
-        openai_api_key: Optional[str] = None,
-        openai_api_version: Optional[str] = None,
-        azure_endpoint: Optional[str] = None,
-        azure_deployment: Optional[str] = None,
         model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        deployment_name: Optional[str] = None,
+        api_version: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "AzureOpenAIEmbeddingModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if api_key is None and "api_key" in alias_values:
+            api_key = alias_values["api_key"]
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+        if deployment_name is None and "deployment_name" in alias_values:
+            deployment_name = alias_values["deployment_name"]
+
         settings = get_settings()
 
-        if openai_api_key is not None:
+        if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.openai_api_key: SecretStr | None = SecretStr(openai_api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
-            self.openai_api_key = settings.AZURE_OPENAI_API_KEY
+            self.api_key = settings.AZURE_OPENAI_API_KEY
+
+        api_version = api_version or settings.OPENAI_API_VERSION
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.AZURE_OPENAI_ENDPOINT is not None:
+            base_url = str(settings.AZURE_OPENAI_ENDPOINT).rstrip("/")
 
-        self.openai_api_version = (
-            openai_api_version or settings.OPENAI_API_VERSION
+        deployment_name = (
+            deployment_name or settings.AZURE_EMBEDDING_DEPLOYMENT_NAME
         )
-        self.azure_endpoint = (
-            azure_endpoint
-            or settings.AZURE_OPENAI_ENDPOINT
-            and str(settings.AZURE_OPENAI_ENDPOINT)
+
+        model = model or settings.AZURE_EMBEDDING_MODEL_NAME or deployment_name
+
+        # validation
+        self.deployment_name = require_param(
+            deployment_name,
+            provider_label="AzureOpenAIEmbeddingModel",
+            env_var_name="AZURE_EMBEDDING_DEPLOYMENT_NAME",
+            param_hint="deployment_name",
+        )
+
+        self.base_url = require_param(
+            base_url,
+            provider_label="AzureOpenAIEmbeddingModel",
+            env_var_name="AZURE_OPENAI_ENDPOINT",
+            param_hint="base_url",
         )
 
-        self.azure_deployment = (
-            azure_deployment or settings.AZURE_EMBEDDING_DEPLOYMENT_NAME
+        self.api_version = require_param(
+            api_version,
+            provider_label="AzureOpenAIEmbeddingModel",
+            env_var_name="OPENAI_API_VERSION",
+            param_hint="api_version",
         )
-        self.client_kwargs = client_kwargs or {}
-        self.model_name = model or self.azure_deployment
+
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(self.model_name)
+        super().__init__(model)
 
     @retry_azure
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=text, model=self.model_name, **self.generation_kwargs
+            input=text, model=self.name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -63,7 +109,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=texts, model=self.model_name, **self.generation_kwargs
+            input=texts, model=self.name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -71,7 +117,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=text, model=self.model_name, **self.generation_kwargs
+            input=text, model=self.name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -79,13 +125,10 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=texts, model=self.model_name, **self.generation_kwargs
+            input=texts, model=self.name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
-    def get_model_name(self) -> str:
-        return self.model_name
-
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(AzureOpenAI)
@@ -93,21 +136,21 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
     def _build_client(self, cls):
         api_key = require_secret_api_key(
-            self.openai_api_key,
+            self.api_key,
             provider_label="AzureOpenAI",
             env_var_name="AZURE_OPENAI_API_KEY",
-            param_hint="`openai_api_key` to AzureOpenAIEmbeddingModel(...)",
+            param_hint="`api_key` to AzureOpenAIEmbeddingModel(...)",
         )
 
-        client_kwargs = self.client_kwargs.copy()
+        client_kwargs = self.kwargs.copy()
         if not sdk_retries_for(PS.AZURE):
             client_kwargs["max_retries"] = 0
 
         client_init_kwargs = dict(
             api_key=api_key,
-            api_version=self.openai_api_version,
-            azure_endpoint=self.azure_endpoint,
-            azure_deployment=self.azure_deployment,
+            api_version=self.api_version,
+            azure_endpoint=self.base_url,
+            azure_deployment=self.deployment_name,
             **client_kwargs,
         )
         try:
@@ -118,3 +161,6 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
                 client_init_kwargs.pop("max_retries", None)
                 return cls(**client_init_kwargs)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Azure)"

deepeval/models/embedding_models/local_embedding_model.py

@@ -3,14 +3,16 @@ from typing import Dict, List, Optional
 from pydantic import SecretStr
 
 from deepeval.config.settings import get_settings
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseEmbeddingModel
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.utils import require_param
 
 # consistent retry rules
 retry_local = create_retry_decorator(PS.LOCAL)
@@ -19,34 +21,51 @@ retry_local = create_retry_decorator(PS.LOCAL)
 class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def __init__(
         self,
+        model: Optional[str] = None,
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
-        model: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
+
         settings = get_settings()
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = get_settings().LOCAL_EMBEDDING_API_KEY
 
-        self.base_url = (
-            base_url
-            or settings.LOCAL_EMBEDDING_BASE_URL
-            and str(settings.LOCAL_EMBEDDING_BASE_URL)
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.LOCAL_EMBEDDING_BASE_URL is not None:
+            base_url = str(settings.LOCAL_EMBEDDING_BASE_URL).rstrip("/")
+
+        model = model or settings.LOCAL_EMBEDDING_MODEL_NAME
+        # validation
+        model = require_param(
+            model,
+            provider_label="LocalEmbeddingModel",
+            env_var_name="LOCAL_EMBEDDING_MODEL_NAME",
+            param_hint="model",
+        )
+
+        self.base_url = require_param(
+            base_url,
+            provider_label="LocalEmbeddingModel",
+            env_var_name="LOCAL_EMBEDDING_BASE_URL",
+            param_hint="base_url",
         )
-        self.model_name = model or settings.LOCAL_EMBEDDING_MODEL_NAME
-        self.client_kwargs = client_kwargs or {}
+
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(self.model_name)
+        super().__init__(model)
 
     @retry_local
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
-            model=self.model_name, input=[text], **self.generation_kwargs
+            model=self.name, input=[text], **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -54,7 +73,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return [data.embedding for data in response.data]
 
@@ -62,7 +81,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
-            model=self.model_name, input=[text], **self.generation_kwargs
+            model=self.name, input=[text], **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -70,7 +89,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return [data.embedding for data in response.data]
 
@@ -78,9 +97,6 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     # Model
     ###############################################
 
-    def get_model_name(self):
-        return self.model_name
-
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(OpenAI)
@@ -94,7 +110,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
             param_hint="`api_key` to LocalEmbeddingModel(...)",
         )
 
-        client_kwargs = self.client_kwargs.copy()
+        client_kwargs = self.kwargs.copy()
         if not sdk_retries_for(PS.LOCAL):
             client_kwargs["max_retries"] = 0
 
@@ -111,3 +127,6 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
                 client_init_kwargs.pop("max_retries", None)
                 return cls(**client_init_kwargs)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Local Model)"

deepeval/models/embedding_models/ollama_embedding_model.py

@@ -1,42 +1,69 @@
-from ollama import Client, AsyncClient
 from typing import List, Optional, Dict
 
 from deepeval.config.settings import get_settings
+from deepeval.utils import require_dependency
 from deepeval.models import DeepEvalBaseEmbeddingModel
+from deepeval.models.utils import (
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.utils import require_param
 
 retry_ollama = create_retry_decorator(PS.OLLAMA)
 
+_ALIAS_MAP = {"base_url": ["host"]}
+
 
 class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def __init__(
         self,
         model: Optional[str] = None,
-        host: Optional[str] = None,
+        base_url: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "OllamaEmbeddingModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+
         settings = get_settings()
 
-        self.host = (
-            host
-            or settings.LOCAL_EMBEDDING_BASE_URL
-            and str(settings.LOCAL_EMBEDDING_BASE_URL)
+        if base_url is not None:
+            self.base_url = str(base_url).rstrip("/")
+        elif settings.LOCAL_EMBEDDING_BASE_URL is not None:
+            self.base_url = str(settings.LOCAL_EMBEDDING_BASE_URL).rstrip("/")
+        else:
+            self.base_url = "http://localhost:11434"
+
+        model = model or settings.LOCAL_EMBEDDING_MODEL_NAME
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="OllamaEmbeddingModel",
+            env_var_name="LOCAL_EMBEDDING_MODEL_NAME",
+            param_hint="model",
         )
-        self.model_name = model or settings.LOCAL_EMBEDDING_MODEL_NAME
-        self.client_kwargs = client_kwargs or {}
+
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(self.model_name)
+        super().__init__(model)
 
     @retry_ollama
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
-            model=self.model_name, input=text, **self.generation_kwargs
+            model=self.name, input=text, **self.generation_kwargs
         )
         return response["embeddings"][0]
 
@@ -44,7 +71,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return response["embeddings"]
 
@@ -52,7 +79,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
-            model=self.model_name, input=text, **self.generation_kwargs
+            model=self.name, input=text, **self.generation_kwargs
         )
         return response["embeddings"][0]
 
@@ -60,7 +87,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return response["embeddings"]
 
@@ -69,12 +96,18 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     ###############################################
 
     def load_model(self, async_mode: bool = False):
+        ollama = require_dependency(
+            "ollama",
+            provider_label="OllamaEmbeddingModel",
+            install_hint="Install it with `pip install ollama`.",
+        )
+
         if not async_mode:
-            return self._build_client(Client)
-        return self._build_client(AsyncClient)
+            return self._build_client(ollama.Client)
+        return self._build_client(ollama.AsyncClient)
 
     def _build_client(self, cls):
-        return cls(host=self.host, **self.client_kwargs)
+        return cls(host=self.base_url, **self.kwargs)
 
     def get_model_name(self):
-        return f"{self.model_name} (Ollama)"
+        return f"{self.name} (Ollama)"