deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/config/utils.py

@@ -1,7 +1,8 @@
 import json
 import os
 import re
-
+from dotenv import dotenv_values
+from pathlib import Path
 from typing import Any, Iterable, List, Optional
 
 
@@ -142,3 +143,10 @@ def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
 def constrain_between(value: float, lo: float, hi: float) -> float:
     """Return value constrained to the inclusive range [lo, hi]."""
     return min(max(value, lo), hi)
+
+
+def read_dotenv_file(path: Path) -> dict[str, str]:
+    if not path.exists():
+        return {}
+    values = dotenv_values(path)
+    return {key: value for key, value in values.items() if value is not None}

deepeval/dataset/api.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from typing import Optional, List
 
 from deepeval.dataset.golden import Golden, ConversationalGolden
@@ -11,6 +11,17 @@ class APIDataset(BaseModel):
         None, alias="conversationalGoldens"
     )
 
+    @model_validator(mode="after")
+    def set_image_mappings_for_goldens(self):
+        if self.goldens:
+            for golden in self.goldens:
+                golden.images_mapping = golden._get_images_mapping()
+        if self.conversational_goldens:
+            for golden in self.conversational_goldens:
+                golden.images_mapping = golden._get_images_mapping()
+
+        return self
+
 
 class APIQueueDataset(BaseModel):
     alias: str
@@ -19,6 +30,17 @@ class APIQueueDataset(BaseModel):
         None, alias="conversationalGoldens"
     )
 
+    @model_validator(mode="after")
+    def set_image_mappings_for_goldens(self):
+        if self.goldens:
+            for golden in self.goldens:
+                golden.images_mapping = golden._get_images_mapping()
+        if self.conversational_goldens:
+            for golden in self.conversational_goldens:
+                golden.images_mapping = golden._get_images_mapping()
+
+        return self
+
 
 class DatasetHttpResponse(BaseModel):
     id: str

deepeval/dataset/golden.py

@@ -1,6 +1,8 @@
+import re
 from pydantic import BaseModel, Field, PrivateAttr, model_validator
 from typing import Optional, Dict, List
 from deepeval.test_case import ToolCall, Turn, MLLMImage
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 
 
 class Golden(BaseModel):
@@ -33,6 +35,9 @@ class Golden(BaseModel):
         default=None, serialization_alias="customColumnKeyValues"
     )
     multimodal: bool = Field(False, exclude=True)
+    images_mapping: Dict[str, MLLMImage] = Field(
+        default=None, alias="imagesMapping"
+    )
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
@@ -45,27 +50,60 @@ class Golden(BaseModel):
             return self
 
         pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
-        self.multimodal = (
+        auto_detect = (
             any(
                 [
-                    (
-                        re.search(pattern, self.input) is not None
-                        if self.input
-                        else False
-                    ),
-                    (
-                        re.search(pattern, self.actual_output) is not None
-                        if self.actual_output
-                        else False
-                    ),
+                    re.search(pattern, self.input or "") is not None,
+                    re.search(pattern, self.actual_output or "") is not None,
                 ]
             )
             if isinstance(self.input, str)
             else self.multimodal
         )
+        if self.retrieval_context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.retrieval_context
+            )
+        if self.context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.context
+            )
+
+        self.multimodal = auto_detect
 
         return self
 
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.input)
+        extract_ids_from_string(self.actual_output)
+        extract_ids_from_string(self.expected_output)
+        extract_ids_from_list(self.context)
+        extract_ids_from_list(self.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None
+
 
 class ConversationalGolden(BaseModel):
     scenario: str
@@ -86,6 +124,9 @@ class ConversationalGolden(BaseModel):
     )
     turns: Optional[List[Turn]] = Field(default=None)
     multimodal: bool = Field(False, exclude=True)
+    images_mapping: Dict[str, MLLMImage] = Field(
+        default=None, alias="imagesMapping"
+    )
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
@@ -98,15 +139,59 @@ class ConversationalGolden(BaseModel):
             return self
 
         pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
-        self.multimodal = (
-            any(
-                [
-                    re.search(pattern, turn.content) is not None
-                    for turn in self.turns
-                ]
-            )
-            if self.turns
-            else self.multimodal
-        )
+        if self.scenario:
+            if re.search(pattern, self.scenario) is not None:
+                self.multimodal = True
+                return self
+        if self.expected_outcome:
+            if re.search(pattern, self.expected_outcome) is not None:
+                self.multimodal = True
+                return self
+        if self.user_description:
+            if re.search(pattern, self.user_description) is not None:
+                self.multimodal = True
+                return self
+        if self.turns:
+            for turn in self.turns:
+                if re.search(pattern, turn.content) is not None:
+                    self.multimodal = True
+                    return self
+                if turn.retrieval_context is not None:
+                    self.multimodal = any(
+                        re.search(pattern, context) is not None
+                        for context in turn.retrieval_context
+                    )
 
         return self
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.scenario)
+        extract_ids_from_string(self.expected_outcome)
+        extract_ids_from_list(self.context)
+        extract_ids_from_string(self.user_description)
+        if self.turns:
+            for turn in self.turns:
+                extract_ids_from_string(turn.content)
+                extract_ids_from_list(turn.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/evaluate/evaluate.py

@@ -46,7 +46,6 @@ from deepeval.telemetry import capture_evaluation_run
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
 )
 from deepeval.metrics.indicator import (
     format_metric_description,
@@ -75,7 +74,6 @@ def assert_test(
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     golden: Optional[Golden] = None,
@@ -190,7 +188,6 @@ def evaluate(
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     # Evals on Confident AI