deepeval 3.7.5__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.7.5"
+__version__: str = "3.7.6"

deepeval/config/settings.py

@@ -27,6 +27,7 @@ from pydantic import (
     field_validator,
     model_validator,
     SecretStr,
+    PositiveFloat,
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing import Any, Dict, List, Optional, NamedTuple
@@ -317,6 +318,19 @@ class Settings(BaseSettings):
 
     # Anthropic
     ANTHROPIC_API_KEY: Optional[SecretStr] = None
+    ANTHROPIC_MODEL_NAME: Optional[str] = None
+    ANTHROPIC_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = None
+    ANTHROPIC_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = None
+
+    # AWS
+    AWS_ACCESS_KEY_ID: Optional[SecretStr] = None
+    AWS_SECRET_ACCESS_KEY: Optional[SecretStr] = None
+    # AWS Bedrock
+    USE_AWS_BEDROCK_MODEL: Optional[bool] = None
+    AWS_BEDROCK_MODEL_NAME: Optional[str] = None
+    AWS_BEDROCK_REGION: Optional[str] = None
+    AWS_BEDROCK_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = None
+    AWS_BEDROCK_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = None
     # Azure Open AI
     AZURE_OPENAI_API_KEY: Optional[SecretStr] = None
     AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None
@@ -329,6 +343,8 @@ class Settings(BaseSettings):
     USE_DEEPSEEK_MODEL: Optional[bool] = None
     DEEPSEEK_API_KEY: Optional[SecretStr] = None
     DEEPSEEK_MODEL_NAME: Optional[str] = None
+    DEEPSEEK_COST_PER_INPUT_TOKEN: Optional[float] = None
+    DEEPSEEK_COST_PER_OUTPUT_TOKEN: Optional[float] = None
     # Gemini
     USE_GEMINI_MODEL: Optional[bool] = None
     GOOGLE_API_KEY: Optional[SecretStr] = None
@@ -336,11 +352,13 @@ class Settings(BaseSettings):
     GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None
     GOOGLE_CLOUD_PROJECT: Optional[str] = None
     GOOGLE_CLOUD_LOCATION: Optional[str] = None
-    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[str] = None
+    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[SecretStr] = None
     # Grok
     USE_GROK_MODEL: Optional[bool] = None
     GROK_API_KEY: Optional[SecretStr] = None
     GROK_MODEL_NAME: Optional[str] = None
+    GROK_COST_PER_INPUT_TOKEN: Optional[float] = None
+    GROK_COST_PER_OUTPUT_TOKEN: Optional[float] = None
     # LiteLLM
     USE_LITELLM: Optional[bool] = None
     LITELLM_API_KEY: Optional[SecretStr] = None
@@ -362,6 +380,8 @@ class Settings(BaseSettings):
     USE_MOONSHOT_MODEL: Optional[bool] = None
     MOONSHOT_API_KEY: Optional[SecretStr] = None
     MOONSHOT_MODEL_NAME: Optional[str] = None
+    MOONSHOT_COST_PER_INPUT_TOKEN: Optional[float] = None
+    MOONSHOT_COST_PER_OUTPUT_TOKEN: Optional[float] = None
     # Ollama
     OLLAMA_MODEL_NAME: Optional[str] = None
     # OpenAI
@@ -388,6 +408,7 @@ class Settings(BaseSettings):
 
     # Azure OpenAI
     USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None
+    AZURE_EMBEDDING_MODEL_NAME: Optional[str] = None
     AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None
     # Local
     USE_LOCAL_EMBEDDINGS: Optional[bool] = None
@@ -614,6 +635,7 @@ class Settings(BaseSettings):
         "SKIP_DEEPEVAL_MISSING_PARAMS",
         "TOKENIZERS_PARALLELISM",
         "TRANSFORMERS_NO_ADVISORY_WARNINGS",
+        "USE_AWS_BEDROCK_MODEL",
         "USE_OPENAI_MODEL",
         "USE_AZURE_OPENAI",
         "USE_LOCAL_MODEL",
@@ -647,6 +669,8 @@ class Settings(BaseSettings):
     @field_validator(
         "OPENAI_COST_PER_INPUT_TOKEN",
         "OPENAI_COST_PER_OUTPUT_TOKEN",
+        "AWS_BEDROCK_COST_PER_INPUT_TOKEN",
+        "AWS_BEDROCK_COST_PER_OUTPUT_TOKEN",
         "TEMPERATURE",
         "CONFIDENT_TRACE_SAMPLE_RATE",
         "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE",
@@ -717,6 +741,16 @@ class Settings(BaseSettings):
             return None
         return s.upper()
 
+    @field_validator("AWS_BEDROCK_REGION", mode="before")
+    @classmethod
+    def _normalize_lower(cls, v):
+        if v is None:
+            return None
+        s = str(v).strip()
+        if not s:
+            return None
+        return s.lower()
+
     @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="before")
     @classmethod
     def _coerce_to_list(cls, v):

deepeval/dataset/api.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from typing import Optional, List
 
 from deepeval.dataset.golden import Golden, ConversationalGolden
@@ -11,6 +11,17 @@ class APIDataset(BaseModel):
         None, alias="conversationalGoldens"
     )
 
+    @model_validator(mode="after")
+    def set_image_mappings_for_goldens(self):
+        if self.goldens:
+            for golden in self.goldens:
+                golden.images_mapping = golden._get_images_mapping()
+        if self.conversational_goldens:
+            for golden in self.conversational_goldens:
+                golden.images_mapping = golden._get_images_mapping()
+
+        return self
+
 
 class APIQueueDataset(BaseModel):
     alias: str
@@ -19,6 +30,17 @@ class APIQueueDataset(BaseModel):
         None, alias="conversationalGoldens"
     )
 
+    @model_validator(mode="after")
+    def set_image_mappings_for_goldens(self):
+        if self.goldens:
+            for golden in self.goldens:
+                golden.images_mapping = golden._get_images_mapping()
+        if self.conversational_goldens:
+            for golden in self.conversational_goldens:
+                golden.images_mapping = golden._get_images_mapping()
+
+        return self
+
 
 class DatasetHttpResponse(BaseModel):
     id: str

deepeval/dataset/golden.py

@@ -1,6 +1,8 @@
+import re
 from pydantic import BaseModel, Field, PrivateAttr, model_validator
 from typing import Optional, Dict, List
 from deepeval.test_case import ToolCall, Turn, MLLMImage
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 
 
 class Golden(BaseModel):
@@ -33,6 +35,9 @@ class Golden(BaseModel):
         default=None, serialization_alias="customColumnKeyValues"
     )
     multimodal: bool = Field(False, exclude=True)
+    images_mapping: Dict[str, MLLMImage] = Field(
+        default=None, alias="imagesMapping"
+    )
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
@@ -45,27 +50,60 @@ class Golden(BaseModel):
             return self
 
         pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
-        self.multimodal = (
+        auto_detect = (
             any(
                 [
-                    (
-                        re.search(pattern, self.input) is not None
-                        if self.input
-                        else False
-                    ),
-                    (
-                        re.search(pattern, self.actual_output) is not None
-                        if self.actual_output
-                        else False
-                    ),
+                    re.search(pattern, self.input or "") is not None,
+                    re.search(pattern, self.actual_output or "") is not None,
                 ]
             )
             if isinstance(self.input, str)
             else self.multimodal
         )
+        if self.retrieval_context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.retrieval_context
+            )
+        if self.context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.context
+            )
+
+        self.multimodal = auto_detect
 
         return self
 
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.input)
+        extract_ids_from_string(self.actual_output)
+        extract_ids_from_string(self.expected_output)
+        extract_ids_from_list(self.context)
+        extract_ids_from_list(self.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None
+
 
 class ConversationalGolden(BaseModel):
     scenario: str
@@ -86,6 +124,9 @@ class ConversationalGolden(BaseModel):
     )
     turns: Optional[List[Turn]] = Field(default=None)
     multimodal: bool = Field(False, exclude=True)
+    images_mapping: Dict[str, MLLMImage] = Field(
+        default=None, alias="imagesMapping"
+    )
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
@@ -98,15 +139,59 @@ class ConversationalGolden(BaseModel):
             return self
 
         pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
-        self.multimodal = (
-            any(
-                [
-                    re.search(pattern, turn.content) is not None
-                    for turn in self.turns
-                ]
-            )
-            if self.turns
-            else self.multimodal
-        )
+        if self.scenario:
+            if re.search(pattern, self.scenario) is not None:
+                self.multimodal = True
+                return self
+        if self.expected_outcome:
+            if re.search(pattern, self.expected_outcome) is not None:
+                self.multimodal = True
+                return self
+        if self.user_description:
+            if re.search(pattern, self.user_description) is not None:
+                self.multimodal = True
+                return self
+        if self.turns:
+            for turn in self.turns:
+                if re.search(pattern, turn.content) is not None:
+                    self.multimodal = True
+                    return self
+                if turn.retrieval_context is not None:
+                    self.multimodal = any(
+                        re.search(pattern, context) is not None
+                        for context in turn.retrieval_context
+                    )
 
         return self
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.scenario)
+        extract_ids_from_string(self.expected_outcome)
+        extract_ids_from_list(self.context)
+        extract_ids_from_string(self.user_description)
+        if self.turns:
+            for turn in self.turns:
+                extract_ids_from_string(turn.content)
+                extract_ids_from_list(turn.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/evaluate/evaluate.py

@@ -46,7 +46,6 @@ from deepeval.telemetry import capture_evaluation_run
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
 )
 from deepeval.metrics.indicator import (
     format_metric_description,
@@ -75,7 +74,6 @@ def assert_test(
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     golden: Optional[Golden] = None,
@@ -190,7 +188,6 @@ def evaluate(
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     # Evals on Confident AI