deepeval 3.4.7__py3-none-any.whl → 3.4.8__py3-none-any.whl

deepeval/cli/utils.py

@@ -1,7 +1,13 @@
-from rich import print
+from __future__ import annotations
+
+import os
 import webbrowser
 import pyfiglet
-from typing import Optional
+
+from enum import Enum
+from pathlib import Path
+from rich import print
+from typing import Optional, Dict, Iterable, List, Tuple, Union
 from opentelemetry.trace import Span
 
 from deepeval.key_handler import (
@@ -14,8 +20,25 @@ from deepeval.test_run.test_run import (
     global_test_run_manager,
 )
 from deepeval.confident.api import get_confident_api_key, set_confident_api_key
+from deepeval.cli.dotenv_handler import DotenvHandler
 
+
+StrOrEnum = Union[str, "Enum"]
 PROD = "https://app.confident-ai.com"
+# List all mutually exclusive USE_* keys
+USE_MODEL_KEYS: List[ModelKeyValues | EmbeddingKeyValues] = [
+    ModelKeyValues.USE_OPENAI_MODEL,
+    ModelKeyValues.USE_AZURE_OPENAI,
+    ModelKeyValues.USE_LOCAL_MODEL,
+    ModelKeyValues.USE_GROK_MODEL,
+    ModelKeyValues.USE_MOONSHOT_MODEL,
+    ModelKeyValues.USE_DEEPSEEK_MODEL,
+    ModelKeyValues.USE_GEMINI_MODEL,
+    ModelKeyValues.USE_LITELLM,
+    EmbeddingKeyValues.USE_AZURE_OPENAI_EMBEDDING,
+    EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS,
+    # MAINTENANCE: add more if new USE_* keys appear
+]
 
 
 def render_login_message():
@@ -65,3 +88,94 @@ def clear_evaluation_model_keys():
 def clear_embedding_model_keys():
     for key in EmbeddingKeyValues:
         KEY_FILE_HANDLER.remove_key(key)
+
+
+def _to_str_key(k: StrOrEnum) -> str:
+    return k.value if hasattr(k, "value") else str(k)
+
+
+def _normalize_kv(updates: Dict[StrOrEnum, str]) -> Dict[str, str]:
+    return {_to_str_key(k): v for k, v in updates.items()}
+
+
+def _normalize_keys(keys: Iterable[StrOrEnum]) -> list[str]:
+    return [_to_str_key(k) for k in keys]
+
+
+def _parse_save_option(
+    save_opt: str | None, default_path: str = ".env.local"
+) -> Tuple[bool, str | None]:
+    if not save_opt:
+        return False, None
+    kind, *rest = save_opt.split(":", 1)
+    if kind != "dotenv":
+        return False, None
+    path = rest[0] if rest else default_path
+    return True, path
+
+
+def resolve_save_target(save_opt: Optional[str]) -> Optional[str]:
+    """
+    Returns a normalized save target string like 'dotenv:.env.local' or None.
+    Precedence:
+      1) --save=...
+      2) DEEPEVAL_DEFAULT_SAVE (opt-in project default)
+      3) None (no save)
+    """
+    if save_opt:
+        return save_opt
+
+    env_default = os.getenv("DEEPEVAL_DEFAULT_SAVE")
+    if env_default and env_default.strip():
+        return env_default.strip()
+
+    return None
+
+
+def save_environ_to_store(
+    save_opt: str | None, updates: Dict[StrOrEnum, str]
+) -> Tuple[bool, str | None]:
+    """
+    Save 'updates' into the selected store (currently only dotenv). Idempotent upsert.
+    Returns (handled, path).
+    """
+    ok, path = _parse_save_option(save_opt)
+    if not ok:
+        return False, None
+    if updates:
+        DotenvHandler(path).upsert(_normalize_kv(updates))
+    return True, path
+
+
+def unset_environ_in_store(
+    save_opt: str | None, keys: Iterable[StrOrEnum]
+) -> Tuple[bool, str | None]:
+    """
+    Remove keys from the selected store (currently only dotenv).
+    Returns (handled, path).
+    """
+    ok, path = _parse_save_option(save_opt)
+    if not ok:
+        return False, None
+    norm = _normalize_keys(keys)
+    if norm:
+        DotenvHandler(path).unset(norm)
+    return True, path
+
+
+def switch_model_provider(target: ModelKeyValues, save: str = None) -> None:
+    """
+    Ensure exactly one USE_* model flag is set to "YES" and the rest to "NO",
+    both in the .deepeval json store and in a dotenv file (if save is provided).
+    """
+    if target not in USE_MODEL_KEYS:
+        raise ValueError(f"{target} is not a recognized USE_* model key")
+
+    for key in USE_MODEL_KEYS:
+        value = "YES" if key == target else "NO"
+        KEY_FILE_HANDLER.write_key(key, value)
+
+        if save:
+            handled, path = save_environ_to_store(save, {key: value})
+            if not handled:
+                print("Unsupported --save option. Use --save=dotenv[:path].")

deepeval/key_handler.py

@@ -1,12 +1,42 @@
 """File for handling API key"""
 
+import os
 import json
+import logging
+
 from enum import Enum
 from typing import Union
 
 from .constants import KEY_FILE, HIDDEN_DIR
 
 
+logger = logging.getLogger(__name__)
+
+
+SECRET_KEYS = {
+    # General providers
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    # Azure OpenAI
+    "AZURE_OPENAI_API_KEY",
+    # Google / Gemini
+    "GOOGLE_API_KEY",
+    # xAI Grok
+    "GROK_API_KEY",
+    # Moonshot
+    "MOONSHOT_API_KEY",
+    # DeepSeek
+    "DEEPSEEK_API_KEY",
+    # LiteLLM
+    "LITELLM_API_KEY",
+    # Local gateways (if any require keys)
+    "LOCAL_MODEL_API_KEY",
+    "LOCAL_EMBEDDING_API_KEY",
+}
+
+_WARNED_SECRET_KEYS = set()
+
+
 class KeyValues(Enum):
     # Confident AI
     API_KEY = "api_key"
@@ -79,10 +109,21 @@ class KeyFileHandler:
     def __init__(self):
         self.data = {}
 
+    def _ensure_dir(self):
+        os.makedirs(HIDDEN_DIR, exist_ok=True)
+
     def write_key(
         self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues], value
     ):
         """Appends or updates data in the hidden file"""
+
+        # hard stop on secrets: never write to disk
+        if key.value in SECRET_KEYS:
+            logger.warning(
+                f"{key} is blacklisted, refusing to persist. Keep your secrets in .env or .env.local instead"
+            )
+            return
+
         try:
             with open(f"{HIDDEN_DIR}/{KEY_FILE}", "r") as f:
                 # Load existing data
@@ -99,13 +140,15 @@ class KeyFileHandler:
         self.data[key.value] = value
 
         # Write the updated data back to the file
+        self._ensure_dir()
         with open(f"{HIDDEN_DIR}/{KEY_FILE}", "w") as f:
             json.dump(self.data, f)
 
     def fetch_data(
         self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
     ):
-        """Fetches the data from the hidden file"""
+        """Fetches the data from the hidden file.
+        NOTE: secrets in this file are deprecated; prefer env/.env."""
         try:
             with open(f"{HIDDEN_DIR}/{KEY_FILE}", "r") as f:
                 try:
@@ -116,7 +159,24 @@ class KeyFileHandler:
         except FileNotFoundError:
             # Handle the case when the file doesn't exist
             self.data = {}
-        return self.data.get(key.value)
+
+        value = self.data.get(key.value)
+
+        # Deprecation: warn only if we're actually returning a secret
+        if (
+            value is not None
+            and key.value in SECRET_KEYS
+            and key.value not in _WARNED_SECRET_KEYS
+        ):
+            logger.warning(
+                f"Reading secret '{key.value}' from legacy {HIDDEN_DIR}/{KEY_FILE}. "
+                "Persisting API keys in plaintext is deprecated. "
+                "Move this to your environment (.env / .env.local). "
+                "This fallback will be removed in a future release."
+            )
+            _WARNED_SECRET_KEYS.add(key.value)
+
+        return value
 
     def remove_key(
         self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
@@ -130,6 +190,7 @@ class KeyFileHandler:
                     # Handle corrupted JSON file
                     self.data = {}
             self.data.pop(key.value, None)  # Remove the key if it exists
+            self._ensure_dir()
             with open(f"{HIDDEN_DIR}/{KEY_FILE}", "w") as f:
                 json.dump(self.data, f)
         except FileNotFoundError:

deepeval/metrics/__init__.py

@@ -5,7 +5,8 @@ from .base_metric import (
     BaseArenaMetric,
 )
 
-from .dag.dag import DAGMetric
+from .dag.dag import DAGMetric, DeepAcyclicGraph
+from .conversational_dag.conversational_dag import ConversationalDAGMetric
 from .bias.bias import BiasMetric
 from .toxicity.toxicity import ToxicityMetric
 from .pii_leakage.pii_leakage import PIILeakageMetric
@@ -67,6 +68,8 @@ __all__ = [
     "ArenaGEval",
     "ConversationalGEval",
     "DAGMetric",
+    "DeepAcyclicGraph",
+    "ConversationalDAGMetric"
     # RAG metrics
     "AnswerRelevancyMetric",
     "FaithfulnessMetric",

deepeval/metrics/conversational_dag/__init__.py

@@ -0,0 +1,7 @@
+from .nodes import (
+    ConversationalBaseNode,
+    ConversationalVerdictNode,
+    ConversationalTaskNode,
+    ConversationalBinaryJudgementNode,
+    ConversationalNonBinaryJudgementNode,
+)

deepeval/metrics/conversational_dag/conversational_dag.py

@@ -0,0 +1,139 @@
+from typing import Optional, Union
+from deepeval.metrics import BaseConversationalMetric
+from deepeval.test_case import (
+    ConversationalTestCase,
+)
+from deepeval.utils import get_or_create_event_loop
+from deepeval.metrics.utils import (
+    check_conversational_test_case_params,
+    construct_verbose_logs,
+    initialize_model,
+)
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.g_eval.schema import *
+from deepeval.metrics import DeepAcyclicGraph
+from deepeval.metrics.dag.utils import (
+    is_valid_dag_from_roots,
+    extract_required_params,
+    copy_graph,
+)
+
+
+class ConversationalDAGMetric(BaseConversationalMetric):
+
+    def __init__(
+        self,
+        name: str,
+        dag: DeepAcyclicGraph,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        threshold: float = 0.5,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+        _include_dag_suffix: bool = True,
+    ):
+        if (
+            is_valid_dag_from_roots(
+                root_nodes=dag.root_nodes, multiturn=dag.multiturn
+            )
+            == False
+        ):
+            raise ValueError("Cycle detected in DAG graph.")
+
+        self._verbose_steps: List[str] = []
+        self.dag = copy_graph(dag)
+        self.name = name
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.threshold = 1 if strict_mode else threshold
+        self.include_reason = include_reason
+        self.strict_mode = strict_mode
+        self.async_mode = async_mode
+        self.verbose_mode = verbose_mode
+        self._include_dag_suffix = _include_dag_suffix
+
+    def measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        check_conversational_test_case_params(
+            test_case,
+            extract_required_params(self.dag.root_nodes, multiturn=True),
+            self,
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                    )
+                )
+            else:
+                self.dag._execute(metric=self, test_case=test_case)
+                self.success = self.is_successful()
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        *self._verbose_steps,
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        check_conversational_test_case_params(
+            test_case,
+            extract_required_params(self.dag.root_nodes, multiturn=True),
+            self,
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            await self.dag._a_execute(metric=self, test_case=test_case)
+            self.success = self.is_successful()
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    *self._verbose_steps,
+                    f"Score: {self.score}\nReason: {self.reason}",
+                ],
+            )
+            return self.score
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        if self._include_dag_suffix:
+            return f"{self.name} [ConversationalDAG]"
+        else:
+            return self.name