deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.7.3"
+__version__: str = "3.7.5"

deepeval/cli/test.py

@@ -160,7 +160,7 @@ def run(
         pytest_args.extend(["--identifier", identifier])
 
     # Add the deepeval plugin file to pytest arguments
-    pytest_args.extend(["-p", "plugins"])
+    pytest_args.extend(["-p", "deepeval"])
     # Append the extra arguments collected by allow_extra_args=True
     # Pytest will raise its own error if the arguments are invalid (error:
     if ctx.args:

deepeval/config/settings.py

@@ -49,6 +49,8 @@ _DEPRECATED_TO_OVERRIDE = {
     "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS": "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE",
     "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS": "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE",
 }
+# Track which secrets we've warned about when loading from the legacy keyfile
+_LEGACY_KEYFILE_SECRET_WARNED: set[str] = set()
 
 
 def _find_legacy_enum(env_key: str):
@@ -88,6 +90,82 @@ def _is_secret_key(settings: "Settings", env_key: str) -> bool:
     return False
 
 
+def _merge_legacy_keyfile_into_env() -> None:
+    """
+    Backwards compatibility: merge values from the legacy .deepeval/.deepeval
+    JSON keystore into os.environ for known Settings fields, without
+    overwriting existing process env vars.
+
+    This runs before we compute the Settings env fingerprint so that Pydantic
+    can see these values on first construction.
+
+    Precedence: process env -> dotenv -> legacy json
+    """
+    # if somebody really wants to skip this behavior
+    if parse_bool(os.getenv("DEEPEVAL_DISABLE_LEGACY_KEYFILE"), default=False):
+        return
+
+    from deepeval.constants import HIDDEN_DIR, KEY_FILE
+    from deepeval.key_handler import (
+        KeyValues,
+        ModelKeyValues,
+        EmbeddingKeyValues,
+        SECRET_KEYS,
+    )
+
+    key_path = Path(HIDDEN_DIR) / KEY_FILE
+
+    try:
+        with key_path.open("r", encoding="utf-8") as f:
+            try:
+                data = json.load(f)
+            except json.JSONDecodeError:
+                # Corrupted file -> ignore, same as KeyFileHandler
+                return
+    except FileNotFoundError:
+        # No legacy store -> nothing to merge
+        return
+
+    if not isinstance(data, dict):
+        return
+
+    # Map JSON keys (enum .value) -> env keys (enum .name)
+    mapping: Dict[str, str] = {}
+    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):
+        for member in enum:
+            mapping[member.value] = member.name
+
+    for json_key, raw in data.items():
+        env_key = mapping.get(json_key)
+        if not env_key:
+            continue
+
+        # Process env always wins
+        if env_key in os.environ:
+            continue
+        if raw is None:
+            continue
+
+        # Mirror the legacy warning semantics for secrets, but only once per key
+        if (
+            json_key in SECRET_KEYS
+            and json_key not in _LEGACY_KEYFILE_SECRET_WARNED
+        ):
+            logger.warning(
+                "Reading secret '%s' from legacy %s/%s. "
+                "Persisting API keys in plaintext is deprecated. "
+                "Move this to your environment (.env / .env.local). "
+                "This fallback will be removed in a future release.",
+                json_key,
+                HIDDEN_DIR,
+                KEY_FILE,
+            )
+            _LEGACY_KEYFILE_SECRET_WARNED.add(json_key)
+
+        # Let Settings validators coerce types; we just inject the raw string
+        os.environ[env_key] = str(raw)
+
+
 def _read_env_file(path: Path) -> Dict[str, str]:
     if not path.exists():
         return {}
@@ -258,6 +336,7 @@ class Settings(BaseSettings):
     GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None
     GOOGLE_CLOUD_PROJECT: Optional[str] = None
     GOOGLE_CLOUD_LOCATION: Optional[str] = None
+    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[str] = None
     # Grok
     USE_GROK_MODEL: Optional[bool] = None
     GROK_API_KEY: Optional[SecretStr] = None
@@ -291,6 +370,12 @@ class Settings(BaseSettings):
     OPENAI_MODEL_NAME: Optional[str] = None
     OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None
     OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None
+    # PortKey
+    USE_PORTKEY_MODEL: Optional[bool] = None
+    PORTKEY_API_KEY: Optional[SecretStr] = None
+    PORTKEY_MODEL_NAME: Optional[str] = None
+    PORTKEY_BASE_URL: Optional[AnyUrl] = None
+    PORTKEY_PROVIDER_NAME: Optional[str] = None
     # Vertex AI
     VERTEX_AI_MODEL_NAME: Optional[str] = None
     # VLLM
@@ -516,29 +601,30 @@ class Settings(BaseSettings):
         "CONFIDENT_OPEN_BROWSER",
         "CONFIDENT_TRACE_FLUSH",
         "CONFIDENT_TRACE_VERBOSE",
+        "CUDA_LAUNCH_BLOCKING",
+        "DEEPEVAL_VERBOSE_MODE",
+        "DEEPEVAL_GRPC_LOGGING",
+        "DEEPEVAL_DISABLE_DOTENV",
+        "DEEPEVAL_TELEMETRY_OPT_OUT",
+        "DEEPEVAL_UPDATE_WARNING_OPT_IN",
+        "ENABLE_DEEPEVAL_CACHE",
+        "ERROR_REPORTING",
+        "GOOGLE_GENAI_USE_VERTEXAI",
+        "IGNORE_DEEPEVAL_ERRORS",
+        "SKIP_DEEPEVAL_MISSING_PARAMS",
+        "TOKENIZERS_PARALLELISM",
+        "TRANSFORMERS_NO_ADVISORY_WARNINGS",
         "USE_OPENAI_MODEL",
         "USE_AZURE_OPENAI",
         "USE_LOCAL_MODEL",
         "USE_GEMINI_MODEL",
-        "GOOGLE_GENAI_USE_VERTEXAI",
         "USE_MOONSHOT_MODEL",
         "USE_GROK_MODEL",
         "USE_DEEPSEEK_MODEL",
         "USE_LITELLM",
         "USE_AZURE_OPENAI_EMBEDDING",
         "USE_LOCAL_EMBEDDINGS",
-        "DEEPEVAL_GRPC_LOGGING",
-        "DEEPEVAL_DISABLE_DOTENV",
-        "DEEPEVAL_TELEMETRY_OPT_OUT",
-        "DEEPEVAL_UPDATE_WARNING_OPT_IN",
-        "TOKENIZERS_PARALLELISM",
-        "TRANSFORMERS_NO_ADVISORY_WARNINGS",
-        "CUDA_LAUNCH_BLOCKING",
-        "ERROR_REPORTING",
-        "IGNORE_DEEPEVAL_ERRORS",
-        "SKIP_DEEPEVAL_MISSING_PARAMS",
-        "DEEPEVAL_VERBOSE_MODE",
-        "ENABLE_DEEPEVAL_CACHE",
+        "USE_PORTKEY_MODEL",
         mode="before",
     )
     @classmethod
@@ -1008,6 +1094,9 @@ _settings_lock = threading.RLock()
 
 
 def _calc_env_fingerprint() -> str:
+    # Pull legacy .deepeval JSON-based settings into the process env before hashing
+    _merge_legacy_keyfile_into_env()
+
     env = os.environ.copy()
     # must hash in a stable order.
     keys = sorted(

deepeval/dataset/golden.py

@@ -1,6 +1,6 @@
-from pydantic import BaseModel, Field, PrivateAttr
+from pydantic import BaseModel, Field, PrivateAttr, model_validator
 from typing import Optional, Dict, List
-from deepeval.test_case import ToolCall, Turn
+from deepeval.test_case import ToolCall, Turn, MLLMImage
 
 
 class Golden(BaseModel):
@@ -32,10 +32,40 @@ class Golden(BaseModel):
     custom_column_key_values: Optional[Dict[str, str]] = Field(
         default=None, serialization_alias="customColumnKeyValues"
     )
+    multimodal: bool = Field(False, exclude=True)
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        self.multimodal = (
+            any(
+                [
+                    (
+                        re.search(pattern, self.input) is not None
+                        if self.input
+                        else False
+                    ),
+                    (
+                        re.search(pattern, self.actual_output) is not None
+                        if self.actual_output
+                        else False
+                    ),
+                ]
+            )
+            if isinstance(self.input, str)
+            else self.multimodal
+        )
+
+        return self
+
 
 class ConversationalGolden(BaseModel):
     scenario: str
@@ -55,6 +85,28 @@ class ConversationalGolden(BaseModel):
         default=None, serialization_alias="customColumnKeyValues"
     )
     turns: Optional[List[Turn]] = Field(default=None)
+    multimodal: bool = Field(False, exclude=True)
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
+
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        self.multimodal = (
+            any(
+                [
+                    re.search(pattern, turn.content) is not None
+                    for turn in self.turns
+                ]
+            )
+            if self.turns
+            else self.multimodal
+        )
+
+        return self

deepeval/evaluate/configs.py

@@ -7,7 +7,7 @@ from deepeval.test_run.test_run import TestRunResultDisplay
 @dataclass
 class AsyncConfig:
     run_async: bool = True
-    throttle_value: int = 0
+    throttle_value: float = 0
     max_concurrent: int = 20
 
     def __post_init__(self):

deepeval/evaluate/evaluate.py

@@ -54,7 +54,6 @@ from deepeval.metrics.indicator import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
 )
 from deepeval.test_run import (
     global_test_run_manager,
@@ -71,9 +70,7 @@ from deepeval.evaluate.execute import (
 
 
 def assert_test(
-    test_case: Optional[
-        Union[LLMTestCase, ConversationalTestCase, MLLMTestCase]
-    ] = None,
+    test_case: Optional[Union[LLMTestCase, ConversationalTestCase]] = None,
     metrics: Optional[
         Union[
             List[BaseMetric],
@@ -175,7 +172,7 @@ def assert_test(
                 try:
                     if not metric_data.success:
                         failed_metrics_data.append(metric_data)
-                except:
+                except Exception:
                     failed_metrics_data.append(metric_data)
 
         failed_metrics_str = ", ".join(
@@ -188,9 +185,7 @@ def assert_test(
 
 
 def evaluate(
-    test_cases: Union[
-        List[LLMTestCase], List[ConversationalTestCase], List[MLLMTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
     metrics: Optional[
         Union[
             List[BaseMetric],
@@ -272,6 +267,19 @@ def evaluate(
         test_run.hyperparameters = process_hyperparameters(hyperparameters)
         test_run.prompts = process_prompts(hyperparameters)
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
+
+        # In CLI mode (`deepeval test run`), the CLI owns finalization and will
+        # call `wrap_up_test_run()` once after pytest finishes. Finalizing here
+        # as well would double finalize the run and consequently result in
+        # duplicate uploads / local saves and temp file races, so only
+        # do it when we're NOT in CLI mode.
+        if get_is_running_deepeval():
+            return EvaluationResult(
+                test_results=test_results,
+                confident_link=None,
+                test_run_id=None,
+            )
+
         res = global_test_run_manager.wrap_up_test_run(
             run_duration, display_table=False
         )

deepeval/evaluate/execute.py

@@ -58,6 +58,13 @@ from deepeval.metrics import (
     BaseConversationalMetric,
     BaseMultimodalMetric,
     TaskCompletionMetric,
+    # RAG metrics that support both single-turn and multimodal
+    ContextualPrecisionMetric,
+    ContextualRecallMetric,
+    ContextualRelevancyMetric,
+    AnswerRelevancyMetric,
+    FaithfulnessMetric,
+    ToolCorrectnessMetric,
 )
 from deepeval.metrics.indicator import (
     measure_metrics_with_indicator,
@@ -70,7 +77,6 @@ from deepeval.models.retry_policy import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
 )
 from deepeval.test_case.api import create_api_test_case
 from deepeval.test_run import (
@@ -110,6 +116,15 @@ from deepeval.test_run.hyperparameters import (
 
 logger = logging.getLogger(__name__)
 
+MLLM_SUPPORTED_METRICS = [
+    ContextualPrecisionMetric,
+    ContextualRecallMetric,
+    ContextualRelevancyMetric,
+    AnswerRelevancyMetric,
+    FaithfulnessMetric,
+    ToolCorrectnessMetric,
+]
+
 
 def _skip_metrics_for_error(
     span: Optional[BaseSpan] = None,
@@ -263,9 +278,7 @@ async def _await_with_outer_deadline(obj, *args, timeout: float, **kwargs):
 
 
 def execute_test_cases(
-    test_cases: Union[
-        List[LLMTestCase], List[ConversationalTestCase], List[MLLMTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
     metrics: Union[
         List[BaseMetric],
         List[BaseConversationalMetric],
@@ -307,6 +320,8 @@ def execute_test_cases(
         metric.async_mode = False
         if isinstance(metric, BaseMetric):
             llm_metrics.append(metric)
+            if type(metric) in MLLM_SUPPORTED_METRICS:
+                mllm_metrics.append(metric)
         elif isinstance(metric, BaseConversationalMetric):
             conversational_metrics.append(metric)
         elif isinstance(metric, BaseMultimodalMetric):
@@ -325,12 +340,12 @@ def execute_test_cases(
         )
         for i, test_case in enumerate(test_cases):
             # skip what we know we won't run
-            if isinstance(test_case, LLMTestCase):
+            if isinstance(test_case, LLMTestCase) and not test_case.multimodal:
                 if not llm_metrics:
                     update_pbar(progress, pbar_id)
                     continue
                 per_case_total = len(llm_metrics)
-            elif isinstance(test_case, MLLMTestCase):
+            elif isinstance(test_case, LLMTestCase) and test_case.multimodal:
                 if not mllm_metrics:
                     update_pbar(progress, pbar_id)
                     continue
@@ -349,10 +364,16 @@ def execute_test_cases(
 
             metrics_for_case = (
                 llm_metrics
-                if isinstance(test_case, LLMTestCase)
+                if (
+                    isinstance(test_case, LLMTestCase)
+                    and not test_case.multimodal
+                )
                 else (
                     mllm_metrics
-                    if isinstance(test_case, MLLMTestCase)
+                    if (
+                        isinstance(test_case, LLMTestCase)
+                        and test_case.multimodal
+                    )
                     else conversational_metrics
                 )
             )
@@ -360,10 +381,16 @@ def execute_test_cases(
                 test_case=test_case,
                 index=(
                     llm_test_case_count + 1
-                    if isinstance(test_case, LLMTestCase)
+                    if (
+                        isinstance(test_case, LLMTestCase)
+                        and not test_case.multimodal
+                    )
                     else (
                         mllm_test_case_count + 1
-                        if isinstance(test_case, MLLMTestCase)
+                        if (
+                            isinstance(test_case, LLMTestCase)
+                            and test_case.multimodal
+                        )
                         else conversational_test_case_count + 1
                     )
                 ),
@@ -383,7 +410,10 @@ def execute_test_cases(
                         for metric in metrics:
                             metric.error = None  # Reset metric error
 
-                        if isinstance(test_case, LLMTestCase):
+                        if (
+                            isinstance(test_case, LLMTestCase)
+                            and not test_case.multimodal
+                        ):
                             llm_test_case_count += 1
                             cached_test_case = None
                             if cache_config.use_cache:
@@ -436,7 +466,10 @@ def execute_test_cases(
                                 update_pbar(progress, pbar_test_case_id)
 
                         # No caching and not sending test cases to Confident AI for multimodal metrics yet
-                        elif isinstance(test_case, MLLMTestCase):
+                        elif (
+                            isinstance(test_case, LLMTestCase)
+                            and test_case.multimodal
+                        ):
                             mllm_test_case_count += 1
                             for metric in mllm_metrics:
                                 current_index = index_of[id(metric)]
@@ -560,9 +593,7 @@ def execute_test_cases(
 
 
 async def a_execute_test_cases(
-    test_cases: Union[
-        List[LLMTestCase], List[ConversationalTestCase], List[MLLMTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
     metrics: Union[
         List[BaseMetric],
         List[BaseConversationalMetric],
@@ -605,6 +636,8 @@ async def a_execute_test_cases(
     for metric in metrics:
         if isinstance(metric, BaseMetric):
             llm_metrics.append(metric)
+            if type(metric) in MLLM_SUPPORTED_METRICS:
+                mllm_metrics.append(metric)
         elif isinstance(metric, BaseMultimodalMetric):
             mllm_metrics.append(metric)
         elif isinstance(metric, BaseConversationalMetric):
@@ -613,7 +646,7 @@ async def a_execute_test_cases(
     llm_test_case_counter = -1
     mllm_test_case_counter = -1
     conversational_test_case_counter = -1
-    test_results: List[Union[TestResult, MLLMTestCase]] = []
+    test_results: List[Union[TestResult, LLMTestCase]] = []
     tasks = []
 
     if display_config.show_indicator and _use_bar_indicator:
@@ -632,7 +665,10 @@ async def a_execute_test_cases(
         with progress:
             for test_case in test_cases:
                 with capture_evaluation_run("test case"):
-                    if isinstance(test_case, LLMTestCase):
+                    if (
+                        isinstance(test_case, LLMTestCase)
+                        and not test_case.multimodal
+                    ):
                         if len(llm_metrics) == 0:
                             update_pbar(progress, pbar_id)
                             continue
@@ -660,7 +696,10 @@ async def a_execute_test_cases(
                         )
                         tasks.append(asyncio.create_task(task))
 
-                    elif isinstance(test_case, MLLMTestCase):
+                    elif (
+                        isinstance(test_case, LLMTestCase)
+                        and test_case.multimodal
+                    ):
                         mllm_test_case_counter += 1
                         copied_multimodal_metrics: List[
                             BaseMultimodalMetric
@@ -718,11 +757,16 @@ async def a_execute_test_cases(
                     "Gather timed out after %.1fs. Some metrics may be marked as timed out.",
                     _gather_timeout(),
                 )
+                if not error_config.ignore_errors:
+                    raise
 
     else:
         for test_case in test_cases:
             with capture_evaluation_run("test case"):
-                if isinstance(test_case, LLMTestCase):
+                if (
+                    isinstance(test_case, LLMTestCase)
+                    and not test_case.multimodal
+                ):
                     if len(llm_metrics) == 0:
                         continue
                     llm_test_case_counter += 1
@@ -770,7 +814,9 @@ async def a_execute_test_cases(
                     )
                     tasks.append(asyncio.create_task((task)))
 
-                elif isinstance(test_case, MLLMTestCase):
+                elif (
+                    isinstance(test_case, LLMTestCase) and test_case.multimodal
+                ):
                     mllm_test_case_counter += 1
                     copied_multimodal_metrics: List[BaseMultimodalMetric] = (
                         copy_metrics(mllm_metrics)
@@ -803,7 +849,8 @@ async def a_execute_test_cases(
                 if not t.done():
                     t.cancel()
             await asyncio.gather(*tasks, return_exceptions=True)
-            raise
+            if not error_config.ignore_errors:
+                raise
 
     return test_results
 
@@ -812,7 +859,7 @@ async def _a_execute_llm_test_cases(
     metrics: List[BaseMetric],
     test_case: LLMTestCase,
     test_run_manager: TestRunManager,
-    test_results: List[Union[TestResult, MLLMTestCase]],
+    test_results: List[Union[TestResult, LLMTestCase]],
     count: int,
     test_run: TestRun,
     ignore_errors: bool,
@@ -931,9 +978,9 @@ async def _a_execute_llm_test_cases(
 
 async def _a_execute_mllm_test_cases(
     metrics: List[BaseMultimodalMetric],
-    test_case: MLLMTestCase,
+    test_case: LLMTestCase,
     test_run_manager: TestRunManager,
-    test_results: List[Union[TestResult, MLLMTestCase]],
+    test_results: List[Union[TestResult, LLMTestCase]],
     count: int,
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -1010,7 +1057,7 @@ async def _a_execute_conversational_test_cases(
     ],
     test_case: ConversationalTestCase,
     test_run_manager: TestRunManager,
-    test_results: List[Union[TestResult, MLLMTestCase]],
+    test_results: List[Union[TestResult, LLMTestCase]],
     count: int,
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -1773,7 +1820,7 @@ async def a_execute_agentic_test_cases(
 async def _a_execute_agentic_test_case(
     golden: Golden,
     test_run_manager: TestRunManager,
-    test_results: List[Union[TestResult, MLLMTestCase]],
+    test_results: List[Union[TestResult, LLMTestCase]],
     count: int,
     verbose_mode: Optional[bool],
     ignore_errors: bool,
@@ -3202,7 +3249,7 @@ async def _evaluate_test_case_pairs(
 
 def _execute_metric(
     metric: BaseMetric,
-    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
     show_metric_indicator: bool,
     in_component: bool,
     error_config: ErrorConfig,