deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/embedding_models/ollama_embedding_model.py

@@ -1,8 +1,11 @@
-from ollama import Client, AsyncClient
 from typing import List, Optional, Dict
 
-from deepeval.key_handler import EmbeddingKeyValues, KEY_FILE_HANDLER
+from deepeval.config.settings import get_settings
+from deepeval.utils import require_dependency
 from deepeval.models import DeepEvalBaseEmbeddingModel
+from deepeval.models.utils import (
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
@@ -11,30 +14,45 @@ from deepeval.constants import ProviderSlug as PS
 
 retry_ollama = create_retry_decorator(PS.OLLAMA)
 
+_ALIAS_MAP = {"base_url": ["host"]}
+
 
 class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def __init__(
         self,
         model: Optional[str] = None,
-        host: Optional[str] = None,
+        base_url: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
-        self.host = host or KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_BASE_URL
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "OllamaEmbeddingModel",
+            kwargs,
+            _ALIAS_MAP,
         )
-        self.model_name = model or KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_MODEL_NAME
+
+        # re-map depricated keywords to re-named positional args
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+
+        settings = get_settings()
+
+        self.base_url = (
+            base_url
+            or settings.LOCAL_EMBEDDING_BASE_URL
+            and str(settings.LOCAL_EMBEDDING_BASE_URL)
         )
-        self.client_kwargs = client_kwargs or {}
+        model = model or settings.LOCAL_EMBEDDING_MODEL_NAME
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(self.model_name)
+        super().__init__(model)
 
     @retry_ollama
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
-            model=self.model_name, input=text, **self.generation_kwargs
+            model=self.name, input=text, **self.generation_kwargs
         )
         return response["embeddings"][0]
 
@@ -42,7 +60,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return response["embeddings"]
 
@@ -50,7 +68,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
-            model=self.model_name, input=text, **self.generation_kwargs
+            model=self.name, input=text, **self.generation_kwargs
         )
         return response["embeddings"][0]
 
@@ -58,7 +76,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
-            model=self.model_name, input=texts, **self.generation_kwargs
+            model=self.name, input=texts, **self.generation_kwargs
         )
         return response["embeddings"]
 
@@ -67,12 +85,18 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     ###############################################
 
     def load_model(self, async_mode: bool = False):
+        ollama = require_dependency(
+            "ollama",
+            provider_label="OllamaEmbeddingModel",
+            install_hint="Install it with `pip install ollama`.",
+        )
+
         if not async_mode:
-            return self._build_client(Client)
-        return self._build_client(AsyncClient)
+            return self._build_client(ollama.Client)
+        return self._build_client(ollama.AsyncClient)
 
     def _build_client(self, cls):
-        return cls(host=self.host, **self.client_kwargs)
+        return cls(host=self.base_url, **self.kwargs)
 
     def get_model_name(self):
-        return f"{self.model_name} (Ollama)"
+        return f"{self.name} (Ollama)"

deepeval/models/embedding_models/openai_embedding_model.py

@@ -1,5 +1,12 @@
 from typing import Dict, Optional, List
 from openai import OpenAI, AsyncOpenAI
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.models import DeepEvalBaseEmbeddingModel
 from deepeval.models.retry_policy import (
     create_retry_decorator,
@@ -15,32 +22,53 @@ valid_openai_embedding_models = [
     "text-embedding-3-large",
     "text-embedding-ada-002",
 ]
+
 default_openai_embedding_model = "text-embedding-3-small"
 
+_ALIAS_MAP = {
+    "api_key": ["openai_api_key"],
+}
+
 
 class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
     def __init__(
         self,
         model: Optional[str] = None,
-        openai_api_key: Optional[str] = None,
+        api_key: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
-        **client_kwargs,
+        **kwargs,
     ):
-        self.openai_api_key = openai_api_key
-        self.model_name = model if model else default_openai_embedding_model
-        if self.model_name not in valid_openai_embedding_models:
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "OpenAIEmbeddingModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if api_key is None and "api_key" in alias_values:
+            api_key = alias_values["api_key"]
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = get_settings().OPENAI_API_KEY
+
+        model = model if model else default_openai_embedding_model
+        if model not in valid_openai_embedding_models:
             raise ValueError(
                 f"Invalid model. Available OpenAI Embedding models: {', '.join(valid_openai_embedding_models)}"
             )
-        self.client_kwargs = client_kwargs or {}
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
+        super().__init__(model)
 
     @retry_openai
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=text, model=self.model_name, **self.generation_kwargs
+            input=text, model=self.name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -48,7 +76,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=texts, model=self.model_name, **self.generation_kwargs
+            input=texts, model=self.name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -56,7 +84,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=text, model=self.model_name, **self.generation_kwargs
+            input=text, model=self.name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -64,7 +92,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=texts, model=self.model_name, **self.generation_kwargs
+            input=texts, model=self.name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -72,21 +100,25 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     # Model
     ###############################################
 
-    def get_model_name(self):
-        return self.model_name
-
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(OpenAI)
         return self._build_client(AsyncOpenAI)
 
     def _build_client(self, cls):
-        client_kwargs = self.client_kwargs.copy()
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="OpenAI",
+            env_var_name="OPENAI_API_KEY",
+            param_hint="`api_key` to OpenAIEmbeddingModel(...)",
+        )
+
+        client_kwargs = self.kwargs.copy()
         if not sdk_retries_for(PS.OPENAI):
             client_kwargs["max_retries"] = 0
 
         client_init_kwargs = dict(
-            api_key=self.openai_api_key,
+            api_key=api_key,
             **client_kwargs,
         )
         try:
@@ -97,3 +129,6 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
                 client_init_kwargs.pop("max_retries", None)
                 return cls(**client_init_kwargs)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (OpenAI)"

deepeval/models/llms/amazon_bedrock_model.py

@@ -1,5 +1,3 @@
-import asyncio
-
 from typing import Optional, Tuple, Union, Dict
 from contextlib import AsyncExitStack
 from pydantic import BaseModel
@@ -76,6 +74,7 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         try:
             payload = self.get_converse_request_body(prompt)
             client = await self._ensure_client()

deepeval/models/llms/anthropic_model.py

@@ -1,8 +1,7 @@
 import warnings
 
 from typing import Optional, Tuple, Union, Dict
-from anthropic import Anthropic, AsyncAnthropic
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
@@ -10,10 +9,13 @@ from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
-from deepeval.models.utils import parse_model_name
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.utils import require_dependency
 
 # consistent retry rules
 retry_anthropic = create_retry_decorator(PS.ANTHROPIC)
@@ -30,26 +32,44 @@ model_pricing = {
     "claude-instant-1.2": {"input": 0.80 / 1e6, "output": 2.40 / 1e6},
 }
 
+_ALIAS_MAP = {
+    "api_key": ["_anthropic_api_key"],
+}
+
 
 class AnthropicModel(DeepEvalBaseLLM):
     def __init__(
         self,
         model: str = "claude-3-7-sonnet-latest",
+        api_key: Optional[str] = None,
         temperature: float = 0,
-        _anthropic_api_key: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = parse_model_name(model)
-        self._anthropic_api_key = _anthropic_api_key
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "AnthropicModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if api_key is None and "api_key" in alias_values:
+            api_key = alias_values["api_key"]
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = get_settings().ANTHROPIC_API_KEY
 
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
 
-        self.kwargs = kwargs
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Generate functions
@@ -59,6 +79,7 @@ class AnthropicModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         chat_model = self.load_model()
         message = chat_model.messages.create(
             max_tokens=1024,
@@ -68,7 +89,7 @@ class AnthropicModel(DeepEvalBaseLLM):
                     "content": prompt,
                 }
             ],
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -85,6 +106,7 @@ class AnthropicModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[str, float]:
+
         chat_model = self.load_model(async_mode=True)
         message = await chat_model.messages.create(
             max_tokens=1024,
@@ -94,7 +116,7 @@ class AnthropicModel(DeepEvalBaseLLM):
                     "content": prompt,
                 }
             ],
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -113,7 +135,7 @@ class AnthropicModel(DeepEvalBaseLLM):
     ###############################################
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        pricing = model_pricing.get(self.model_name)
+        pricing = model_pricing.get(self.name)
 
         if pricing is None:
             # Calculate average cost from all known models
@@ -126,7 +148,7 @@ class AnthropicModel(DeepEvalBaseLLM):
             pricing = {"input": avg_input_cost, "output": avg_output_cost}
 
             warnings.warn(
-                f"[Warning] Pricing not defined for model '{self.model_name}'. "
+                f"[Warning] Pricing not defined for model '{self.name}'. "
                 "Using average input/output token costs from existing model_pricing."
             )
 
@@ -139,12 +161,15 @@ class AnthropicModel(DeepEvalBaseLLM):
     ###############################################
 
     def load_model(self, async_mode: bool = False):
-        if not async_mode:
-            return self._build_client(Anthropic)
-        return self._build_client(AsyncAnthropic)
+        module = require_dependency(
+            "anthropic",
+            provider_label="AnthropicModel",
+            install_hint="Install it with `pip install anthropic`.",
+        )
 
-    def get_model_name(self):
-        return f"{self.model_name}"
+        if not async_mode:
+            return self._build_client(module.Anthropic)
+        return self._build_client(module.AsyncAnthropic)
 
     def _client_kwargs(self) -> Dict:
         kwargs = dict(self.kwargs or {})
@@ -155,9 +180,14 @@ class AnthropicModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
-        settings = get_settings()
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Anthropic",
+            env_var_name="ANTHROPIC_API_KEY",
+            param_hint="`api_key` to AnthropicModel(...)",
+        )
         kw = dict(
-            api_key=settings.ANTHROPIC_API_KEY or self._anthropic_api_key,
+            api_key=api_key,
             **self._client_kwargs(),
         )
         try:
@@ -168,3 +198,6 @@ class AnthropicModel(DeepEvalBaseLLM):
                 kw.pop("max_retries", None)
                 return cls(**kw)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Anthropic)"