deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/test_case/arena_test_case.py

@@ -1,7 +1,7 @@
 from typing import List, Dict, Optional, Union
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pydantic import BaseModel
-
+import re
 from deepeval.test_case import (
     LLMTestCase,
 )
@@ -19,6 +19,7 @@ class Contestant(BaseModel):
 @dataclass
 class ArenaTestCase:
     contestants: List[Contestant]
+    multimodal: bool = field(default=False)
 
     def __post_init__(self):
         contestant_names = [contestant.name for contestant in self.contestants]
@@ -38,6 +39,10 @@ class ArenaTestCase:
                     "All contestants must have the same 'expected_output'."
                 )
 
+        for contestant in self.contestants:
+            if contestant.test_case.multimodal:
+                self.multimodal = True
+
 
 class Arena:
     test_cases: List[ArenaTestCase]

deepeval/test_case/conversational_test_case.py

@@ -1,3 +1,4 @@
+import re
 from pydantic import (
     BaseModel,
     Field,
@@ -17,6 +18,7 @@ from deepeval.test_case.mcp import (
     MCPToolCall,
     validate_mcp_servers,
 )
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 
 
 class TurnParams(Enum):
@@ -170,12 +172,28 @@ class ConversationalTestCase(BaseModel):
             return self
 
         pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
-        self.multimodal = any(
-            [
-                re.search(pattern, turn.content) is not None
-                for turn in self.turns
-            ]
-        )
+        if self.scenario:
+            if re.search(pattern, self.scenario) is not None:
+                self.multimodal = True
+                return self
+        if self.expected_outcome:
+            if re.search(pattern, self.expected_outcome) is not None:
+                self.multimodal = True
+                return self
+        if self.user_description:
+            if re.search(pattern, self.user_description) is not None:
+                self.multimodal = True
+                return self
+        if self.turns:
+            for turn in self.turns:
+                if re.search(pattern, turn.content) is not None:
+                    self.multimodal = True
+                    return self
+                if turn.retrieval_context is not None:
+                    self.multimodal = any(
+                        re.search(pattern, context) is not None
+                        for context in turn.retrieval_context
+                    )
 
         return self
 
@@ -215,3 +233,34 @@ class ConversationalTestCase(BaseModel):
         data["turns"] = copied_turns
 
         return data
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.scenario)
+        extract_ids_from_string(self.expected_outcome)
+        extract_ids_from_list(self.context)
+        extract_ids_from_string(self.user_description)
+        for turn in self.turns:
+            extract_ids_from_string(turn.content)
+            extract_ids_from_list(turn.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/test_case/llm_test_case.py

@@ -60,19 +60,34 @@ class MLLMImage:
             if self.local:
                 path = self.process_url(self.url)
                 self.filename = os.path.basename(path)
-                self.mimeType = (
-                    mimetypes.guess_type(path)[0] or "application/octet-stream"
-                )
-                with open(path, "rb") as f:
-                    raw = f.read()
-                self.dataBase64 = base64.b64encode(raw).decode("ascii")
+                self.mimeType = mimetypes.guess_type(path)[0] or "image/jpeg"
+
+                if not os.path.exists(path):
+                    raise FileNotFoundError(f"Image file not found: {path}")
+
+                self._load_base64(path)
             else:
+                if not self.url.startswith(("http://", "https://")):
+                    raise ValueError(
+                        f"Invalid remote URL format: {self.url}. URL must start with http:// or https://"
+                    )
                 self.filename = None
                 self.mimeType = None
                 self.dataBase64 = None
 
         _MLLM_IMAGE_REGISTRY[self._id] = self
 
+    def _load_base64(self, path: str):
+        with open(path, "rb") as f:
+            raw = f.read()
+        self.dataBase64 = base64.b64encode(raw).decode("ascii")
+
+    def ensure_images_loaded(self):
+        if self.local and self.dataBase64 is None:
+            path = self.process_url(self.url)
+            self._load_base64(path)
+        return self
+
     def _placeholder(self) -> str:
         return f"[DEEPEVAL:IMAGE:{self._id}]"
 
@@ -376,6 +391,16 @@ class LLMTestCase(BaseModel):
             if isinstance(self.input, str)
             else self.multimodal
         )
+        if self.retrieval_context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.retrieval_context
+            )
+        if self.context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.context
+            )
 
         self.multimodal = auto_detect
         return self
@@ -486,3 +511,32 @@ class LLMTestCase(BaseModel):
                 )
 
         return data
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.input)
+        extract_ids_from_string(self.actual_output)
+        extract_ids_from_string(self.expected_output)
+        extract_ids_from_list(self.context)
+        extract_ids_from_list(self.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/test_run/api.py

@@ -126,6 +126,9 @@ class ConversationalApiTestCase(BaseModel):
     additional_metadata: Optional[Dict] = Field(
         None, alias="additionalMetadata"
     )
+    images_mapping: Optional[Dict[str, MLLMImage]] = Field(
+        None, alias="imagesMapping"
+    )
     tags: Optional[List[str]] = Field(None)
 
     def update_metric_data(self, metrics_data: MetricData):

deepeval/test_run/test_run.py

@@ -1028,8 +1028,13 @@ class TestRunManager:
                 LATEST_TEST_RUN_FILE_PATH,
                 save_under_key=LATEST_TEST_RUN_DATA_KEY,
             )
+            token_cost = (
+                f"{test_run.evaluation_cost} USD"
+                if test_run.evaluation_cost
+                else "None"
+            )
             console.print(
-                f"\n\n[rgb(5,245,141)]✓[/rgb(5,245,141)] Evaluation completed 🎉! (time taken: {round(runDuration, 2)}s | token cost: {test_run.evaluation_cost} USD)\n"
+                f"\n\n[rgb(5,245,141)]✓[/rgb(5,245,141)] Evaluation completed 🎉! (time taken: {round(runDuration, 2)}s | token cost: {token_cost})\n"
                 f"» Test Results ({test_run.test_passed + test_run.test_failed} total tests):\n",
                 f"  » Pass Rate: {round((test_run.test_passed / (test_run.test_passed + test_run.test_failed)) * 100, 2)}% | Passed: [bold green]{test_run.test_passed}[/bold green] | Failed: [bold red]{test_run.test_failed}[/bold red]\n\n",
                 "=" * 80,

deepeval/utils.py

@@ -268,6 +268,32 @@ def set_should_use_cache(yes: bool):
         s.ENABLE_DEEPEVAL_CACHE = yes
 
 
+###################
+# Timeout Helpers #
+###################
+def are_timeouts_disabled() -> bool:
+    return bool(get_settings().DEEPEVAL_DISABLE_TIMEOUTS)
+
+
+def get_per_task_timeout_seconds() -> float:
+    return get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+
+
+def get_per_task_timeout() -> Optional[float]:
+    return None if are_timeouts_disabled() else get_per_task_timeout_seconds()
+
+
+def get_gather_timeout_seconds() -> float:
+    return (
+        get_per_task_timeout_seconds()
+        + get_settings().DEEPEVAL_TASK_GATHER_BUFFER_SECONDS
+    )
+
+
+def get_gather_timeout() -> Optional[float]:
+    return None if are_timeouts_disabled() else get_gather_timeout_seconds()
+
+
 def login(api_key: str):
     if not api_key or not isinstance(api_key, str):
         raise ValueError("Oh no! Please provide an api key string to login.")

{deepeval-3.7.5.dist-info → deepeval-3.7.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.7.5
+Version: 3.7.7
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -100,9 +100,9 @@ Description-Content-Type: text/markdown
     <a href="https://www.readme-i18n.com/confident-ai/deepeval?lang=zh">中文</a>
 </p>
 
-**DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating and testing large-language model systems. It is similar to Pytest but specialized for unit testing LLM outputs. DeepEval incorporates the latest research to evaluate LLM outputs based on metrics such as G-Eval, hallucination, answer relevancy, RAGAS, etc., which uses LLMs and various other NLP models that runs **locally on your machine** for evaluation.
+**DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating and testing large-language model systems. It is similar to Pytest but specialized for unit testing LLM outputs. DeepEval incorporates the latest research to evaluate LLM outputs based on metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which uses LLM-as-a-judge and other NLP models that runs **locally on your machine** for evaluation.
 
-Whether your LLM applications are RAG pipelines, chatbots, AI agents, implemented via LangChain or LlamaIndex, DeepEval has you covered. With it, you can easily determine the optimal models, prompts, and architecture to improve your RAG pipeline, agentic workflows, prevent prompt drifting, or even transition from OpenAI to hosting your own Deepseek R1 with confidence.
+Whether your LLM applications are AI agents, RAG pipelines, or chatbots, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily determine the optimal models, prompts, and architecture to improve your RAG pipeline, agentic workflows, prevent prompt drifting, or even transition from OpenAI to hosting your own Deepseek R1 with confidence.
 
 > [!IMPORTANT]
 > Need a place for your DeepEval testing data to live 🏡❤️? [Sign up to the DeepEval platform](https://confident-ai.com?utm_source=GitHub) to compare iterations of your LLM app, generate & share testing reports, and more.