deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/models/embedding_models/azure_embedding_model.py

@@ -13,6 +13,7 @@ from deepeval.models.utils import (
     require_secret_api_key,
     normalize_kwargs_and_extract_aliases,
 )
+from deepeval.utils import require_param
 
 
 retry_azure = create_retry_decorator(PS.AZURE)
@@ -31,7 +32,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
         deployment_name: Optional[str] = None,
-        openai_api_version: Optional[str] = None,
+        api_version: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
@@ -53,25 +54,46 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.AZURE_OPENAI_API_KEY
 
-        self.openai_api_version = (
-            openai_api_version or settings.OPENAI_API_VERSION
+        api_version = api_version or settings.OPENAI_API_VERSION
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.AZURE_OPENAI_ENDPOINT is not None:
+            base_url = str(settings.AZURE_OPENAI_ENDPOINT).rstrip("/")
+
+        deployment_name = (
+            deployment_name or settings.AZURE_EMBEDDING_DEPLOYMENT_NAME
+        )
+
+        model = model or settings.AZURE_EMBEDDING_MODEL_NAME or deployment_name
+
+        # validation
+        self.deployment_name = require_param(
+            deployment_name,
+            provider_label="AzureOpenAIEmbeddingModel",
+            env_var_name="AZURE_EMBEDDING_DEPLOYMENT_NAME",
+            param_hint="deployment_name",
         )
-        self.base_url = (
-            base_url
-            or settings.AZURE_OPENAI_ENDPOINT
-            and str(settings.AZURE_OPENAI_ENDPOINT)
+
+        self.base_url = require_param(
+            base_url,
+            provider_label="AzureOpenAIEmbeddingModel",
+            env_var_name="AZURE_OPENAI_ENDPOINT",
+            param_hint="base_url",
         )
 
-        self.deployment_name = (
-            deployment_name or settings.AZURE_EMBEDDING_DEPLOYMENT_NAME
+        self.api_version = require_param(
+            api_version,
+            provider_label="AzureOpenAIEmbeddingModel",
+            env_var_name="OPENAI_API_VERSION",
+            param_hint="api_version",
         )
+
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = normalized_kwargs
-        model = model or self.deployment_name
         self.generation_kwargs = generation_kwargs or {}
         super().__init__(model)
 
@@ -126,7 +148,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
         client_init_kwargs = dict(
             api_key=api_key,
-            api_version=self.openai_api_version,
+            api_version=self.api_version,
             azure_endpoint=self.base_url,
             azure_deployment=self.deployment_name,
             **client_kwargs,

deepeval/models/embedding_models/local_embedding_model.py

@@ -12,7 +12,7 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.utils import require_param
 
 # consistent retry rules
 retry_local = create_retry_decorator(PS.LOCAL)
@@ -31,16 +31,31 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
         settings = get_settings()
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = get_settings().LOCAL_EMBEDDING_API_KEY
 
-        self.base_url = (
-            base_url
-            or settings.LOCAL_EMBEDDING_BASE_URL
-            and str(settings.LOCAL_EMBEDDING_BASE_URL)
-        )
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.LOCAL_EMBEDDING_BASE_URL is not None:
+            base_url = str(settings.LOCAL_EMBEDDING_BASE_URL).rstrip("/")
+
         model = model or settings.LOCAL_EMBEDDING_MODEL_NAME
+        # validation
+        model = require_param(
+            model,
+            provider_label="LocalEmbeddingModel",
+            env_var_name="LOCAL_EMBEDDING_MODEL_NAME",
+            param_hint="model",
+        )
+
+        self.base_url = require_param(
+            base_url,
+            provider_label="LocalEmbeddingModel",
+            env_var_name="LOCAL_EMBEDDING_BASE_URL",
+            param_hint="base_url",
+        )
+
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}

deepeval/models/embedding_models/ollama_embedding_model.py

@@ -10,7 +10,7 @@ from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.utils import require_param
 
 retry_ollama = create_retry_decorator(PS.OLLAMA)
 
@@ -37,12 +37,23 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
         settings = get_settings()
 
-        self.base_url = (
-            base_url
-            or settings.LOCAL_EMBEDDING_BASE_URL
-            and str(settings.LOCAL_EMBEDDING_BASE_URL)
-        )
+        if base_url is not None:
+            self.base_url = str(base_url).rstrip("/")
+        elif settings.LOCAL_EMBEDDING_BASE_URL is not None:
+            self.base_url = str(settings.LOCAL_EMBEDDING_BASE_URL).rstrip("/")
+        else:
+            self.base_url = "http://localhost:11434"
+
         model = model or settings.LOCAL_EMBEDDING_MODEL_NAME
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="OllamaEmbeddingModel",
+            env_var_name="LOCAL_EMBEDDING_MODEL_NAME",
+            param_hint="model",
+        )
+
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}

deepeval/models/embedding_models/openai_embedding_model.py

@@ -2,6 +2,7 @@ from typing import Dict, Optional, List
 from openai import OpenAI, AsyncOpenAI
 from pydantic import SecretStr
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.utils import (
     require_secret_api_key,
@@ -51,13 +52,13 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = get_settings().OPENAI_API_KEY
 
         model = model if model else default_openai_embedding_model
         if model not in valid_openai_embedding_models:
-            raise ValueError(
+            raise DeepEvalError(
                 f"Invalid model. Available OpenAI Embedding models: {', '.join(valid_openai_embedding_models)}"
             )
         self.kwargs = normalized_kwargs

deepeval/models/llms/__init__.py

@@ -9,6 +9,7 @@ from .litellm_model import LiteLLMModel
 from .kimi_model import KimiModel
 from .grok_model import GrokModel
 from .deepseek_model import DeepSeekModel
+from .portkey_model import PortkeyModel
 
 __all__ = [
     "AzureOpenAIModel",
@@ -22,4 +23,5 @@ __all__ = [
     "KimiModel",
     "GrokModel",
     "DeepSeekModel",
+    "PortkeyModel",
 ]

deepeval/models/llms/amazon_bedrock_model.py

@@ -1,130 +1,286 @@
-from typing import Optional, Tuple, Union, Dict
+import base64
+from typing import Optional, Tuple, Union, Dict, List
 from contextlib import AsyncExitStack
-from pydantic import BaseModel
 
+from pydantic import BaseModel, SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.utils import (
+    require_dependency,
+    require_param,
+)
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
+from deepeval.models.llms.constants import BEDROCK_MODELS_DATA
 from deepeval.models.llms.utils import trim_and_load_json, safe_asyncio_run
 from deepeval.constants import ProviderSlug as PS
+from deepeval.models.utils import (
+    require_costs,
+    normalize_kwargs_and_extract_aliases,
+)
 
-# check aiobotocore availability
-try:
-    from aiobotocore.session import get_session
-    from botocore.config import Config
-
-    aiobotocore_available = True
-except ImportError:
-    aiobotocore_available = False
 
-# define retry policy
 retry_bedrock = create_retry_decorator(PS.BEDROCK)
 
-
-def _check_aiobotocore_available():
-    if not aiobotocore_available:
-        raise ImportError(
-            "aiobotocore and botocore are required for this functionality. "
-            "Install them via your package manager (e.g. pip install aiobotocore botocore)"
-        )
+_ALIAS_MAP = {
+    "model": ["model_id"],
+    "region": ["region_name"],
+    "cost_per_input_token": ["input_token_cost"],
+    "cost_per_output_token": ["output_token_cost"],
+}
 
 
 class AmazonBedrockModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        model_id: str,
-        region_name: str,
+        model: Optional[str] = None,
         aws_access_key_id: Optional[str] = None,
         aws_secret_access_key: Optional[str] = None,
-        input_token_cost: float = 0,
-        output_token_cost: float = 0,
+        cost_per_input_token: Optional[float] = None,
+        cost_per_output_token: Optional[float] = None,
+        region: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        _check_aiobotocore_available()
-        super().__init__(model_id)
-
-        self.model_id = model_id
-        self.region_name = region_name
-        self.aws_access_key_id = aws_access_key_id
-        self.aws_secret_access_key = aws_secret_access_key
-        self.input_token_cost = input_token_cost
-        self.output_token_cost = output_token_cost
-
-        # prepare aiobotocore session, config, and async exit stack
-        self._session = get_session()
+        settings = get_settings()
+
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "AmazonBedrockModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # Backwards compatibility for renamed params
+        if model is None and "model" in alias_values:
+            model = alias_values["model"]
+        if (
+            cost_per_input_token is None
+            and "cost_per_input_token" in alias_values
+        ):
+            cost_per_input_token = alias_values["cost_per_input_token"]
+        if (
+            cost_per_output_token is None
+            and "cost_per_output_token" in alias_values
+        ):
+            cost_per_output_token = alias_values["cost_per_output_token"]
+
+        # Secrets: prefer explicit args -> settings -> then AWS default chain
+        if aws_access_key_id is not None:
+            self.aws_access_key_id: Optional[SecretStr] = SecretStr(
+                aws_access_key_id
+            )
+        else:
+            self.aws_access_key_id = settings.AWS_ACCESS_KEY_ID
+
+        if aws_secret_access_key is not None:
+            self.aws_secret_access_key: Optional[SecretStr] = SecretStr(
+                aws_secret_access_key
+            )
+        else:
+            self.aws_secret_access_key = settings.AWS_SECRET_ACCESS_KEY
+
+        # Dependencies: aiobotocore & botocore
+        aiobotocore_session = require_dependency(
+            "aiobotocore.session",
+            provider_label="AmazonBedrockModel",
+            install_hint="Install it with `pip install aiobotocore`.",
+        )
+        self.botocore_module = require_dependency(
+            "botocore",
+            provider_label="AmazonBedrockModel",
+            install_hint="Install it with `pip install botocore`.",
+        )
+        self._session = aiobotocore_session.get_session()
         self._exit_stack = AsyncExitStack()
-        self.kwargs = kwargs
+
+        # Defaults from settings
+        model = model or settings.AWS_BEDROCK_MODEL_NAME
+        region = region or settings.AWS_BEDROCK_REGION
+
+        cost_per_input_token = (
+            cost_per_input_token
+            if cost_per_input_token is not None
+            else settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN
+        )
+        cost_per_output_token = (
+            cost_per_output_token
+            if cost_per_output_token is not None
+            else settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN
+        )
+
+        # Required params
+        model = require_param(
+            model,
+            provider_label="AmazonBedrockModel",
+            env_var_name="AWS_BEDROCK_MODEL_NAME",
+            param_hint="model",
+        )
+        region = require_param(
+            region,
+            provider_label="AmazonBedrockModel",
+            env_var_name="AWS_BEDROCK_REGION",
+            param_hint="region",
+        )
+
+        self.model_data = BEDROCK_MODELS_DATA.get(model)
+        cost_per_input_token, cost_per_output_token = require_costs(
+            self.model_data,
+            model,
+            "AWS_BEDROCK_COST_PER_INPUT_TOKEN",
+            "AWS_BEDROCK_COST_PER_OUTPUT_TOKEN",
+            cost_per_input_token,
+            cost_per_output_token,
+        )
+
+        # Final attributes
+        self.region = region
+        self.cost_per_input_token = float(cost_per_input_token or 0.0)
+        self.cost_per_output_token = float(cost_per_output_token or 0.0)
+
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
         self._client = None
         self._sdk_retry_mode: Optional[bool] = None
 
+        super().__init__(model)
+
     ###############################################
     # Generate functions
     ###############################################
 
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
         return safe_asyncio_run(self.a_generate(prompt, schema))
 
     @retry_bedrock
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
-
-        try:
+    ) -> Tuple[Union[str, BaseModel], float]:
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            payload = self.generate_payload(prompt)
+        else:
             payload = self.get_converse_request_body(prompt)
-            client = await self._ensure_client()
-            response = await client.converse(
-                modelId=self.model_id,
-                messages=payload["messages"],
-                inferenceConfig=payload["inferenceConfig"],
-            )
-            message = response["output"]["message"]["content"][0]["text"]
-            cost = self.calculate_cost(
-                response["usage"]["inputTokens"],
-                response["usage"]["outputTokens"],
-            )
-            if schema is None:
-                return message, cost
-            else:
-                json_output = trim_and_load_json(message)
-                return schema.model_validate(json_output), cost
-        finally:
-            await self.close()
+
+        payload = self.get_converse_request_body(prompt)
+        client = await self._ensure_client()
+        response = await client.converse(
+            modelId=self.get_model_name(),
+            messages=payload["messages"],
+            inferenceConfig=payload["inferenceConfig"],
+        )
+        message = response["output"]["message"]["content"][0]["text"]
+        cost = self.calculate_cost(
+            response["usage"]["inputTokens"],
+            response["usage"]["outputTokens"],
+        )
+        if schema is None:
+            return message, cost
+        else:
+            json_output = trim_and_load_json(message)
+            return schema.model_validate(json_output), cost
+
+    def generate_payload(
+        self, multimodal_input: Optional[List[Union[str, MLLMImage]]] = None
+    ):
+        multimodal_input = [] if multimodal_input is None else multimodal_input
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"text": element})
+            elif isinstance(element, MLLMImage):
+                # Bedrock doesn't support external URLs - must convert everything to bytes
+                element.ensure_images_loaded()
+
+                image_format = (
+                    (element.mimeType or "image/jpeg").split("/")[-1].upper()
+                )
+                image_format = "JPEG" if image_format == "JPG" else image_format
+
+                try:
+                    image_raw_bytes = base64.b64decode(element.dataBase64)
+                except Exception:
+                    raise ValueError(
+                        f"Invalid base64 data in MLLMImage: {element._id}"
+                    )
+
+                content.append(
+                    {
+                        "image": {
+                            "format": image_format,
+                            "source": {"bytes": image_raw_bytes},
+                        }
+                    }
+                )
+
+        return {
+            "messages": [{"role": "user", "content": content}],
+            "inferenceConfig": {
+                **self.generation_kwargs,
+            },
+        }
+
+    #########################
+    # Capabilities          #
+    #########################
+
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        return self.model_data.supports_json
 
     ###############################################
     # Client management
     ###############################################
 
     async def _ensure_client(self):
+
         use_sdk = sdk_retries_for(PS.BEDROCK)
 
         # only rebuild if client is missing or the sdk retry mode changes
         if self._client is None or self._sdk_retry_mode != use_sdk:
-            # Close any previous
-            if self._client is not None:
-                await self._exit_stack.aclose()
-                self._client = None
 
             # create retry config for botocore
             retries_config = {"max_attempts": (5 if use_sdk else 1)}
             if use_sdk:
                 retries_config["mode"] = "adaptive"
 
+            Config = self.botocore_module.config.Config
             config = Config(retries=retries_config)
 
-            cm = self._session.create_client(
-                "bedrock-runtime",
-                region_name=self.region_name,
-                aws_access_key_id=self.aws_access_key_id,
-                aws_secret_access_key=self.aws_secret_access_key,
-                config=config,
+            client_kwargs = {
+                "region_name": self.region,
+                "config": config,
                 **self.kwargs,
-            )
+            }
+
+            if self.aws_access_key_id is not None:
+                client_kwargs["aws_access_key_id"] = (
+                    self.aws_access_key_id.get_secret_value()
+                )
+            if self.aws_secret_access_key is not None:
+                client_kwargs["aws_secret_access_key"] = (
+                    self.aws_secret_access_key.get_secret_value()
+                )
+
+            cm = self._session.create_client("bedrock-runtime", **client_kwargs)
+
             self._client = await self._exit_stack.enter_async_context(cm)
             self._sdk_retry_mode = use_sdk
 
@@ -148,13 +304,13 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
         }
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        return (
-            input_tokens * self.input_token_cost
-            + output_tokens * self.output_token_cost
-        )
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     def load_model(self):
         pass
 
     def get_model_name(self) -> str:
-        return self.model_id
+        return self.name