deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.7.4"
+__version__: str = "3.7.6"

deepeval/config/settings.py

@@ -27,6 +27,7 @@ from pydantic import (
     field_validator,
     model_validator,
     SecretStr,
+    PositiveFloat,
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing import Any, Dict, List, Optional, NamedTuple
@@ -317,6 +318,19 @@ class Settings(BaseSettings):
 
     # Anthropic
     ANTHROPIC_API_KEY: Optional[SecretStr] = None
+    ANTHROPIC_MODEL_NAME: Optional[str] = None
+    ANTHROPIC_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = None
+    ANTHROPIC_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = None
+
+    # AWS
+    AWS_ACCESS_KEY_ID: Optional[SecretStr] = None
+    AWS_SECRET_ACCESS_KEY: Optional[SecretStr] = None
+    # AWS Bedrock
+    USE_AWS_BEDROCK_MODEL: Optional[bool] = None
+    AWS_BEDROCK_MODEL_NAME: Optional[str] = None
+    AWS_BEDROCK_REGION: Optional[str] = None
+    AWS_BEDROCK_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = None
+    AWS_BEDROCK_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = None
     # Azure Open AI
     AZURE_OPENAI_API_KEY: Optional[SecretStr] = None
     AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None
@@ -329,6 +343,8 @@ class Settings(BaseSettings):
     USE_DEEPSEEK_MODEL: Optional[bool] = None
     DEEPSEEK_API_KEY: Optional[SecretStr] = None
     DEEPSEEK_MODEL_NAME: Optional[str] = None
+    DEEPSEEK_COST_PER_INPUT_TOKEN: Optional[float] = None
+    DEEPSEEK_COST_PER_OUTPUT_TOKEN: Optional[float] = None
     # Gemini
     USE_GEMINI_MODEL: Optional[bool] = None
     GOOGLE_API_KEY: Optional[SecretStr] = None
@@ -336,11 +352,13 @@ class Settings(BaseSettings):
     GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None
     GOOGLE_CLOUD_PROJECT: Optional[str] = None
     GOOGLE_CLOUD_LOCATION: Optional[str] = None
-    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[str] = None
+    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[SecretStr] = None
     # Grok
     USE_GROK_MODEL: Optional[bool] = None
     GROK_API_KEY: Optional[SecretStr] = None
     GROK_MODEL_NAME: Optional[str] = None
+    GROK_COST_PER_INPUT_TOKEN: Optional[float] = None
+    GROK_COST_PER_OUTPUT_TOKEN: Optional[float] = None
     # LiteLLM
     USE_LITELLM: Optional[bool] = None
     LITELLM_API_KEY: Optional[SecretStr] = None
@@ -362,6 +380,8 @@ class Settings(BaseSettings):
     USE_MOONSHOT_MODEL: Optional[bool] = None
     MOONSHOT_API_KEY: Optional[SecretStr] = None
     MOONSHOT_MODEL_NAME: Optional[str] = None
+    MOONSHOT_COST_PER_INPUT_TOKEN: Optional[float] = None
+    MOONSHOT_COST_PER_OUTPUT_TOKEN: Optional[float] = None
     # Ollama
     OLLAMA_MODEL_NAME: Optional[str] = None
     # OpenAI
@@ -388,6 +408,7 @@ class Settings(BaseSettings):
 
     # Azure OpenAI
     USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None
+    AZURE_EMBEDDING_MODEL_NAME: Optional[str] = None
     AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None
     # Local
     USE_LOCAL_EMBEDDINGS: Optional[bool] = None
@@ -614,6 +635,7 @@ class Settings(BaseSettings):
         "SKIP_DEEPEVAL_MISSING_PARAMS",
         "TOKENIZERS_PARALLELISM",
         "TRANSFORMERS_NO_ADVISORY_WARNINGS",
+        "USE_AWS_BEDROCK_MODEL",
         "USE_OPENAI_MODEL",
         "USE_AZURE_OPENAI",
         "USE_LOCAL_MODEL",
@@ -647,6 +669,8 @@ class Settings(BaseSettings):
     @field_validator(
         "OPENAI_COST_PER_INPUT_TOKEN",
         "OPENAI_COST_PER_OUTPUT_TOKEN",
+        "AWS_BEDROCK_COST_PER_INPUT_TOKEN",
+        "AWS_BEDROCK_COST_PER_OUTPUT_TOKEN",
         "TEMPERATURE",
         "CONFIDENT_TRACE_SAMPLE_RATE",
         "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE",
@@ -717,6 +741,16 @@ class Settings(BaseSettings):
             return None
         return s.upper()
 
+    @field_validator("AWS_BEDROCK_REGION", mode="before")
+    @classmethod
+    def _normalize_lower(cls, v):
+        if v is None:
+            return None
+        s = str(v).strip()
+        if not s:
+            return None
+        return s.lower()
+
     @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="before")
     @classmethod
     def _coerce_to_list(cls, v):

deepeval/dataset/api.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from typing import Optional, List
 
 from deepeval.dataset.golden import Golden, ConversationalGolden
@@ -11,6 +11,17 @@ class APIDataset(BaseModel):
         None, alias="conversationalGoldens"
     )
 
+    @model_validator(mode="after")
+    def set_image_mappings_for_goldens(self):
+        if self.goldens:
+            for golden in self.goldens:
+                golden.images_mapping = golden._get_images_mapping()
+        if self.conversational_goldens:
+            for golden in self.conversational_goldens:
+                golden.images_mapping = golden._get_images_mapping()
+
+        return self
+
 
 class APIQueueDataset(BaseModel):
     alias: str
@@ -19,6 +30,17 @@ class APIQueueDataset(BaseModel):
         None, alias="conversationalGoldens"
     )
 
+    @model_validator(mode="after")
+    def set_image_mappings_for_goldens(self):
+        if self.goldens:
+            for golden in self.goldens:
+                golden.images_mapping = golden._get_images_mapping()
+        if self.conversational_goldens:
+            for golden in self.conversational_goldens:
+                golden.images_mapping = golden._get_images_mapping()
+
+        return self
+
 
 class DatasetHttpResponse(BaseModel):
     id: str

deepeval/dataset/golden.py

@@ -1,6 +1,8 @@
-from pydantic import BaseModel, Field, PrivateAttr
+import re
+from pydantic import BaseModel, Field, PrivateAttr, model_validator
 from typing import Optional, Dict, List
-from deepeval.test_case import ToolCall, Turn
+from deepeval.test_case import ToolCall, Turn, MLLMImage
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 
 
 class Golden(BaseModel):
@@ -32,10 +34,76 @@ class Golden(BaseModel):
     custom_column_key_values: Optional[Dict[str, str]] = Field(
         default=None, serialization_alias="customColumnKeyValues"
     )
+    multimodal: bool = Field(False, exclude=True)
+    images_mapping: Dict[str, MLLMImage] = Field(
+        default=None, alias="imagesMapping"
+    )
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        auto_detect = (
+            any(
+                [
+                    re.search(pattern, self.input or "") is not None,
+                    re.search(pattern, self.actual_output or "") is not None,
+                ]
+            )
+            if isinstance(self.input, str)
+            else self.multimodal
+        )
+        if self.retrieval_context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.retrieval_context
+            )
+        if self.context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.context
+            )
+
+        self.multimodal = auto_detect
+
+        return self
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.input)
+        extract_ids_from_string(self.actual_output)
+        extract_ids_from_string(self.expected_output)
+        extract_ids_from_list(self.context)
+        extract_ids_from_list(self.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None
+
 
 class ConversationalGolden(BaseModel):
     scenario: str
@@ -55,6 +123,75 @@ class ConversationalGolden(BaseModel):
         default=None, serialization_alias="customColumnKeyValues"
     )
     turns: Optional[List[Turn]] = Field(default=None)
+    multimodal: bool = Field(False, exclude=True)
+    images_mapping: Dict[str, MLLMImage] = Field(
+        default=None, alias="imagesMapping"
+    )
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
+
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        if self.scenario:
+            if re.search(pattern, self.scenario) is not None:
+                self.multimodal = True
+                return self
+        if self.expected_outcome:
+            if re.search(pattern, self.expected_outcome) is not None:
+                self.multimodal = True
+                return self
+        if self.user_description:
+            if re.search(pattern, self.user_description) is not None:
+                self.multimodal = True
+                return self
+        if self.turns:
+            for turn in self.turns:
+                if re.search(pattern, turn.content) is not None:
+                    self.multimodal = True
+                    return self
+                if turn.retrieval_context is not None:
+                    self.multimodal = any(
+                        re.search(pattern, context) is not None
+                        for context in turn.retrieval_context
+                    )
+
+        return self
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.scenario)
+        extract_ids_from_string(self.expected_outcome)
+        extract_ids_from_list(self.context)
+        extract_ids_from_string(self.user_description)
+        if self.turns:
+            for turn in self.turns:
+                extract_ids_from_string(turn.content)
+                extract_ids_from_list(turn.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/evaluate/evaluate.py

@@ -46,7 +46,6 @@ from deepeval.telemetry import capture_evaluation_run
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
 )
 from deepeval.metrics.indicator import (
     format_metric_description,
@@ -54,7 +53,6 @@ from deepeval.metrics.indicator import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
 )
 from deepeval.test_run import (
     global_test_run_manager,
@@ -71,14 +69,11 @@ from deepeval.evaluate.execute import (
 
 
 def assert_test(
-    test_case: Optional[
-        Union[LLMTestCase, ConversationalTestCase, MLLMTestCase]
-    ] = None,
+    test_case: Optional[Union[LLMTestCase, ConversationalTestCase]] = None,
     metrics: Optional[
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     golden: Optional[Golden] = None,
@@ -175,7 +170,7 @@ def assert_test(
                 try:
                     if not metric_data.success:
                         failed_metrics_data.append(metric_data)
-                except:
+                except Exception:
                     failed_metrics_data.append(metric_data)
 
         failed_metrics_str = ", ".join(
@@ -188,14 +183,11 @@ def assert_test(
 
 
 def evaluate(
-    test_cases: Union[
-        List[LLMTestCase], List[ConversationalTestCase], List[MLLMTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
     metrics: Optional[
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     # Evals on Confident AI
@@ -272,6 +264,19 @@ def evaluate(
         test_run.hyperparameters = process_hyperparameters(hyperparameters)
         test_run.prompts = process_prompts(hyperparameters)
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
+
+        # In CLI mode (`deepeval test run`), the CLI owns finalization and will
+        # call `wrap_up_test_run()` once after pytest finishes. Finalizing here
+        # as well would double finalize the run and consequently result in
+        # duplicate uploads / local saves and temp file races, so only
+        # do it when we're NOT in CLI mode.
+        if get_is_running_deepeval():
+            return EvaluationResult(
+                test_results=test_results,
+                confident_link=None,
+                test_run_id=None,
+            )
+
         res = global_test_run_manager.wrap_up_test_run(
             run_duration, display_table=False
         )