deepeval 3.6.9__py3-none-any.whl → 3.7.1__py3-none-any.whl

deepeval/confident/types.py

@@ -1,9 +1,11 @@
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel
 from typing import Any, Optional
 
+from deepeval.utils import make_model_config
+
 
 class ApiResponse(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     success: bool
     data: Optional[Any] = None

deepeval/config/settings.py

@@ -9,10 +9,13 @@ Central config for DeepEval.
   type coercion.
 """
 
+import hashlib
+import json
 import logging
 import math
 import os
 import re
+import threading
 
 from dotenv import dotenv_values
 from pathlib import Path
@@ -22,6 +25,7 @@ from pydantic import (
     confloat,
     conint,
     field_validator,
+    model_validator,
     SecretStr,
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -39,6 +43,13 @@ from deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify
 logger = logging.getLogger(__name__)
 _SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
 
+# settings that were converted to computed fields with override counterparts
+_DEPRECATED_TO_OVERRIDE = {
+    "DEEPEVAL_PER_TASK_TIMEOUT_SECONDS": "DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE",
+    "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS": "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE",
+    "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS": "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE",
+}
+
 
 def _find_legacy_enum(env_key: str):
     from deepeval.key_handler import (
@@ -218,6 +229,11 @@ class Settings(BaseSettings):
     API_KEY: Optional[SecretStr] = None
     CONFIDENT_API_KEY: Optional[SecretStr] = None
 
+    # ======
+    # Base URL for Confident AI API server
+    # ======
+    CONFIDENT_BASE_URL: Optional[str] = None
+
     # General
     TEMPERATURE: Optional[confloat(ge=0, le=2)] = None
 
@@ -690,12 +706,119 @@ class Settings(BaseSettings):
             "CRITICAL, NOTSET, or a numeric logging level."
         )
 
+    @field_validator("DEEPEVAL_TELEMETRY_OPT_OUT", mode="before")
+    @classmethod
+    def _apply_telemetry_enabled_alias(cls, v):
+        """
+        Precedence (most secure):
+        - Any OFF signal wins if both are set:
+          - DEEPEVAL_TELEMETRY_OPT_OUT = truthy  -> OFF
+          - DEEPEVAL_TELEMETRY_ENABLED = falsy   -> OFF
+        - Else, ON signal:
+          - DEEPEVAL_TELEMETRY_OPT_OUT = falsy   -> ON
+          - DEEPEVAL_TELEMETRY_ENABLED = truthy  -> ON
+        - Else None (unset) -> ON
+        """
+
+        def normalize(x):
+            if x is None:
+                return None
+            s = str(x).strip()
+            return None if s == "" else parse_bool(s, default=False)
+
+        new_opt_out = normalize(v)  # True means OFF, False means ON
+        legacy_enabled = normalize(
+            os.getenv("DEEPEVAL_TELEMETRY_ENABLED")
+        )  # True means ON, False means OFF
+
+        off_signal = (new_opt_out is True) or (legacy_enabled is False)
+        on_signal = (new_opt_out is False) or (legacy_enabled is True)
+
+        # Conflict: simultaneous OFF and ON signals
+        if off_signal and on_signal:
+            # Only warn if verbose or debug
+            if parse_bool(
+                os.getenv("DEEPEVAL_VERBOSE_MODE"), default=False
+            ) or logger.isEnabledFor(logging.DEBUG):
+                logger.warning(
+                    "Conflicting telemetry flags detected: DEEPEVAL_TELEMETRY_OPT_OUT=%r, "
+                    "DEEPEVAL_TELEMETRY_ENABLED=%r. Defaulting to OFF.",
+                    new_opt_out,
+                    legacy_enabled,
+                )
+            return True  # OFF wins
+
+        # Clear winner
+        if off_signal:
+            return True  # OFF
+        if on_signal:
+            return False  # ON
+
+        # Unset means ON
+        return False
+
+    @model_validator(mode="after")
+    def _apply_deprecated_computed_env_aliases(self):
+        """
+        Backwards compatibility courtesy:
+        - If users still set a deprecated computed field in the environment,
+          emit a deprecation warning and mirror its value into the matching
+          *_OVERRIDE field (unless the override is already set).
+        - Override always wins if both are present.
+        """
+        for old_key, override_key in _DEPRECATED_TO_OVERRIDE.items():
+            raw = os.getenv(old_key)
+            if raw is None or str(raw).strip() == "":
+                continue
+
+            # if override already set, ignore the deprecated one but log a warning
+            if getattr(self, override_key) is not None:
+                logger.warning(
+                    "Config deprecation: %s is deprecated and was ignored because %s "
+                    "is already set. Please remove %s and use %s going forward.",
+                    old_key,
+                    override_key,
+                    old_key,
+                    override_key,
+                )
+                continue
+
+            # apply the deprecated value into the override field.
+            try:
+                # let pydantic coerce the string to the target type on assignment
+                setattr(self, override_key, raw)
+                logger.warning(
+                    "Config deprecation: %s is deprecated. Its value (%r) was applied to %s. "
+                    "Please migrate to %s and remove %s from your environment.",
+                    old_key,
+                    raw,
+                    override_key,
+                    override_key,
+                    old_key,
+                )
+            except Exception as e:
+                # do not let exception bubble up, just warn
+                logger.warning(
+                    "Config deprecation: %s is deprecated and could not be applied to %s "
+                    "(value=%r): %s",
+                    old_key,
+                    override_key,
+                    raw,
+                    e,
+                )
+        return self
+
     #######################
     # Persistence support #
     #######################
     class _SettingsEditCtx:
+        # TODO: will generate this list in future PR
         COMPUTED_FIELDS: frozenset[str] = frozenset(
-            {"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS"}
+            {
+                "DEEPEVAL_PER_TASK_TIMEOUT_SECONDS",
+                "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS",
+                "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS",
+            }
         )
 
         def __init__(
@@ -880,16 +1003,39 @@ class Settings(BaseSettings):
 
 
 _settings_singleton: Optional[Settings] = None
+_settings_env_fingerprint: "str | None" = None
+_settings_lock = threading.RLock()
+
+
+def _calc_env_fingerprint() -> str:
+    env = os.environ.copy()
+    # must hash in a stable order.
+    keys = sorted(
+        key
+        for key in Settings.model_fields.keys()
+        if key != "_DEPRECATED_TELEMETRY_ENABLED"  # exclude deprecated
+    )
+    # encode as triples: (key, present?, value)
+    items = [(k, k in env, env.get(k)) for k in keys]
+    payload = json.dumps(items, ensure_ascii=False, separators=(",", ":"))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
 
 def get_settings() -> Settings:
-    global _settings_singleton
-    if _settings_singleton is None:
-        _settings_singleton = Settings()
-        from deepeval.config.logging import apply_deepeval_log_level
+    global _settings_singleton, _settings_env_fingerprint
+    fingerprint = _calc_env_fingerprint()
+
+    with _settings_lock:
+        if (
+            _settings_singleton is None
+            or _settings_env_fingerprint != fingerprint
+        ):
+            _settings_singleton = Settings()
+            _settings_env_fingerprint = fingerprint
+            from deepeval.config.logging import apply_deepeval_log_level
 
-        apply_deepeval_log_level()
-    return _settings_singleton
+            apply_deepeval_log_level()
+        return _settings_singleton
 
 
 def reset_settings(*, reload_dotenv: bool = False) -> Settings:
@@ -905,8 +1051,10 @@ def reset_settings(*, reload_dotenv: bool = False) -> Settings:
     Returns:
         The fresh Settings instance.
     """
-    global _settings_singleton
-    if reload_dotenv:
-        autoload_dotenv()
-    _settings_singleton = None
+    global _settings_singleton, _settings_env_fingerprint
+    with _settings_lock:
+        if reload_dotenv:
+            autoload_dotenv()
+        _settings_singleton = None
+        _settings_env_fingerprint = None
     return get_settings()

deepeval/config/settings_manager.py

@@ -4,6 +4,7 @@ dotenv file. Also syncs os.environ, handles unsets, and warns on unknown fields.
 Primary entrypoint: update_settings_and_persist.
 """
 
+import json
 import logging
 import os
 
@@ -33,6 +34,9 @@ def _normalize_for_env(val: Any) -> Optional[str]:
         return val.get_secret_value()
     if isinstance(val, bool):
         return bool_to_env_str(val)
+    # encode sequences as JSON so Settings can parse them back reliably.
+    if isinstance(val, (list, tuple, set)):
+        return json.dumps(list(val))
     return str(val)
 
 

deepeval/evaluate/compare.py

@@ -1,5 +1,6 @@
 from typing import Optional, List, Dict, Callable
 import asyncio
+import time
 from rich.progress import (
     Progress,
     TextColumn,
@@ -8,24 +9,74 @@ from rich.progress import (
     TaskProgressColumn,
 )
 from collections import Counter
+import json
 
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.evaluate.configs import AsyncConfig, DisplayConfig, ErrorConfig
-from deepeval.test_case import ArenaTestCase
+from deepeval.test_case import ArenaTestCase, Contestant
+from deepeval.test_case.api import create_api_test_case
 from deepeval.metrics import ArenaGEval
-from deepeval.utils import add_pbar, update_pbar, custom_console
-from deepeval.utils import get_or_create_event_loop
+from deepeval.utils import (
+    add_pbar,
+    update_pbar,
+    custom_console,
+    get_or_create_event_loop,
+    open_browser,
+)
+from deepeval.test_run.test_run import (
+    TestRun,
+    MetricData,
+    TestRunEncoder,
+    MetricScores,
+    console,
+)
+from deepeval.test_run.hyperparameters import (
+    process_hyperparameters,
+)
+from deepeval.confident.api import Api, Endpoints, HttpMethods, is_confident
 from deepeval.telemetry import capture_evaluation_run
+from deepeval.test_run.api import LLMApiTestCase
+from deepeval.evaluate.utils import create_arena_metric_data
+from deepeval.evaluate.types import PostExperimentRequest
 
 
 def compare(
     test_cases: List[ArenaTestCase],
     metric: ArenaGEval,
+    name: str = "compare()",
     # Configs
     async_config: Optional[AsyncConfig] = AsyncConfig(),
     display_config: Optional[DisplayConfig] = DisplayConfig(),
     error_config: Optional[ErrorConfig] = ErrorConfig(),
 ) -> Dict[str, int]:
+
+    # Prepare test run map
+    unique_contestant_names = set(
+        [
+            contestant.name
+            for test_case in test_cases
+            for contestant in test_case.contestants
+        ]
+    )
+    test_run_map: Dict[str, TestRun] = {}
+    for contestant_name in unique_contestant_names:
+        test_run = TestRun(
+            identifier=contestant_name,
+            test_passed=0,
+            test_failed=0,
+        )
+        test_run.metrics_scores = [
+            MetricScores(
+                metric=metric.name,
+                scores=[],
+                passes=0,
+                fails=0,
+                errors=0,
+            )
+        ]
+        test_run_map[contestant_name] = test_run
+
+    start_time = time.time()
     with capture_evaluation_run("compare()"):
         if async_config.run_async:
             loop = get_or_create_event_loop()
@@ -39,6 +90,7 @@ def compare(
                     throttle_value=async_config.throttle_value,
                     max_concurrent=async_config.max_concurrent,
                     skip_on_missing_params=error_config.skip_on_missing_params,
+                    test_run_map=test_run_map,
                 )
             )
         else:
@@ -49,7 +101,10 @@ def compare(
                 verbose_mode=display_config.verbose_mode,
                 show_indicator=display_config.show_indicator,
                 skip_on_missing_params=error_config.skip_on_missing_params,
+                test_run_map=test_run_map,
             )
+    end_time = time.time()
+    run_duration = end_time - start_time
 
     # Aggregate winners
     winner_counts = Counter()
@@ -57,7 +112,13 @@ def compare(
         if winner:
             winner_counts[winner] += 1
 
-    print(winner_counts)
+    process_test_runs(test_run_map=test_run_map, test_cases=test_cases)
+    wrap_up_experiment(
+        name=name,
+        test_runs=list(test_run_map.values()),
+        winner_counts=winner_counts,
+        run_duration=run_duration,
+    )
     return dict(winner_counts)
 
 
@@ -70,6 +131,7 @@ async def a_execute_arena_test_cases(
     throttle_value: int,
     skip_on_missing_params: bool,
     max_concurrent: int,
+    test_run_map: Dict[str, TestRun],
 ) -> List[str]:
     semaphore = asyncio.Semaphore(max_concurrent)
 
@@ -104,6 +166,8 @@ async def a_execute_arena_test_cases(
                 else metric.verbose_mode
             ),
         )
+
+        start_time = time.perf_counter()
         winner = await _a_handle_metric_measurement(
             metric=metric_copy,
             test_case=test_case,
@@ -112,10 +176,21 @@ async def a_execute_arena_test_cases(
             _progress=progress,
             _pbar_id=pbar_test_case_id,
         )
+        end_time = time.perf_counter()
+        run_duration = end_time - start_time
+
         if winner:
             winners.append(winner)
 
         update_pbar(progress, pbar_id)
+        update_test_run_map(
+            test_case=test_case,
+            index=index,
+            test_run_map=test_run_map,
+            metric_copy=metric_copy,
+            winner=winner,
+            run_duration=run_duration,
+        )
 
     # Create tasks for all test cases
     if show_indicator:
@@ -156,6 +231,7 @@ def execute_arena_test_cases(
     skip_on_missing_params: bool,
     show_indicator: bool,
     verbose_mode: Optional[bool] = None,
+    test_run_map: Optional[Dict[str, TestRun]] = None,
 ) -> List[str]:
     """
     Non-async version of comparing arena test cases.
@@ -183,6 +259,8 @@ def execute_arena_test_cases(
                     else metric.verbose_mode
                 ),
             )
+
+            start_time = time.perf_counter()
             winner = _handle_metric_measurement(
                 metric=metric_copy,
                 test_case=test_case,
@@ -191,10 +269,21 @@ def execute_arena_test_cases(
                 _progress=progress,
                 _pbar_id=pbar_test_case_id,
             )
+            end_time = time.perf_counter()
+            run_duration = end_time - start_time
+
             if winner:
                 winners.append(winner)
 
             update_pbar(progress, pbar_id)
+            update_test_run_map(
+                test_case=test_case,
+                index=i,
+                test_run_map=test_run_map,
+                metric_copy=metric_copy,
+                winner=winner,
+                run_duration=run_duration,
+            )
 
     if show_indicator:
         progress = Progress(
@@ -313,3 +402,125 @@ async def _a_handle_metric_measurement(
                 return None
             else:
                 raise
+
+
+def update_test_run_map(
+    test_case: ArenaTestCase,
+    index: int,
+    test_run_map: Dict[str, TestRun],
+    metric_copy: ArenaGEval,
+    winner: str,
+    run_duration: float,
+):
+    for contestant in test_case.contestants:
+        test_run = test_run_map.get(contestant.name)
+
+        # update test cases in test run
+        api_test_case: LLMApiTestCase = create_api_test_case(
+            test_case=contestant.test_case, index=index
+        )
+        metric_data: MetricData = create_arena_metric_data(
+            metric_copy, contestant.name
+        )
+        api_test_case.update_metric_data(metric_data)
+        api_test_case.update_run_duration(run_duration)
+        test_run.add_test_case(api_test_case)
+
+        # update other test run attributes
+        if test_run.run_duration is None:
+            test_run.run_duration = 0.0
+        test_run.run_duration += run_duration
+
+        # Ensure test_passed and test_failed are initialized
+        if test_run.test_passed is None:
+            test_run.test_passed = 0
+        if test_run.test_failed is None:
+            test_run.test_failed = 0
+
+        if winner == contestant:
+            test_run.test_passed += 1
+        else:
+            test_run.test_failed += 1
+
+        # update metric scores
+        test_run.metrics_scores[0].metric = metric_copy.name
+        test_run.metrics_scores[0].scores.append(
+            1 if winner == contestant else 0
+        )
+        test_run.metrics_scores[0].passes += 1 if winner == contestant else 0
+        test_run.metrics_scores[0].fails += 1 if winner != contestant else 0
+        test_run.metrics_scores[0].errors += 0
+
+
+def process_test_runs(
+    test_run_map: Dict[str, TestRun],
+    test_cases: List[ArenaTestCase],
+):
+    hyperparameters_map = {
+        contestant_name: {} for contestant_name in test_run_map.keys()
+    }
+
+    for test_case in test_cases:
+        for contestant in test_case.contestants:
+            if contestant.hyperparameters:
+                hyperparameters_map[contestant.name].update(
+                    contestant.hyperparameters
+                )
+
+    for contestant_name, hyperparameters in hyperparameters_map.items():
+        test_run = test_run_map.get(contestant_name)
+        test_run.hyperparameters = process_hyperparameters(hyperparameters)
+
+
+def wrap_up_experiment(
+    name: str,
+    test_runs: List[TestRun],
+    winner_counts: Counter,
+    run_duration: float,
+):
+    winner_breakdown = []
+    for contestant, wins in winner_counts.most_common():
+        winner_breakdown.append(
+            f"    » [bold green]{contestant}[/bold green]: {wins} wins"
+        )
+    winner_text = (
+        "\n".join(winner_breakdown) if winner_breakdown else "No winners"
+    )
+    console.print(
+        f"\n🎉 Arena completed! (time taken: {round(run_duration, 2)}s | token cost: {test_runs[0].evaluation_cost if test_runs else 0} USD)\n"
+        f"🏆 Results ({sum(winner_counts.values())} total test cases):\n"
+        f"{winner_text}\n\n"
+    )
+
+    if not is_confident():
+        console.print(
+            f"{'=' * 80}\n"
+            f"\n» Want to share experiments with your team? ❤️ 🏟️\n"
+            f"  » Run [bold]'deepeval login'[/bold] to analyze and save arena results on [rgb(106,0,255)]Confident AI[/rgb(106,0,255)].\n\n"
+        )
+        return
+
+    try:
+        api = Api()
+        experiment_request = PostExperimentRequest(testRuns=test_runs, name=name)
+
+        try:
+            body = experiment_request.model_dump(by_alias=True, exclude_none=True)
+        except AttributeError:
+            body = experiment_request.dict(by_alias=True, exclude_none=True)
+        json_str = json.dumps(body, cls=TestRunEncoder)
+        body = json.loads(json_str)
+
+        _, link = api.send_request(
+            method=HttpMethods.POST,
+            endpoint=Endpoints.EXPERIMENT_ENDPOINT,
+            body=body,
+        )
+        console.print(
+            "[rgb(5,245,141)]✓[/rgb(5,245,141)] Done 🎉! View results on "
+            f"[link={link}]{link}[/link]"
+        )
+        open_browser(link)
+
+    except Exception:
+        raise

deepeval/evaluate/types.py

@@ -4,6 +4,7 @@ from pydantic import BaseModel
 
 from deepeval.test_run.api import MetricData, TurnApi
 from deepeval.test_case import MLLMImage
+from deepeval.test_run import TestRun
 
 
 @dataclass
@@ -29,3 +30,8 @@ class EvaluationResult(BaseModel):
     test_results: List[TestResult]
     confident_link: Optional[str]
     test_run_id: Optional[str]
+
+
+class PostExperimentRequest(BaseModel):
+    testRuns: List[TestRun]
+    name: Optional[str]

deepeval/evaluate/utils.py

@@ -8,6 +8,7 @@ from deepeval.utils import format_turn
 from deepeval.test_run.test_run import TestRunResultDisplay
 from deepeval.dataset import Golden
 from deepeval.metrics import (
+    ArenaGEval,
     BaseMetric,
     BaseConversationalMetric,
     BaseMultimodalMetric,
@@ -84,6 +85,35 @@ def create_metric_data(metric: BaseMetric) -> MetricData:
         )
 
 
+def create_arena_metric_data(metric: ArenaGEval, contestant: str) -> MetricData:
+    if metric.error is not None:
+        return MetricData(
+            name=metric.__name__,
+            threshold=1,
+            score=None,
+            reason=None,
+            success=False,
+            strictMode=True,
+            evaluationModel=metric.evaluation_model,
+            error=metric.error,
+            evaluationCost=metric.evaluation_cost,
+            verboseLogs=metric.verbose_logs,
+        )
+    else:
+        return MetricData(
+            name=metric.__name__,
+            score=1 if contestant == metric.winner else 0,
+            threshold=1,
+            reason=metric.reason,
+            success=metric.is_successful(),
+            strictMode=True,
+            evaluationModel=metric.evaluation_model,
+            error=None,
+            evaluationCost=metric.evaluation_cost,
+            verboseLogs=metric.verbose_logs,
+        )
+
+
 def create_test_result(
     api_test_case: Union[LLMApiTestCase, ConversationalApiTestCase],
 ) -> TestResult:

deepeval/integrations/crewai/handler.py

@@ -23,6 +23,8 @@ try:
         AgentExecutionCompletedEvent,
         ToolUsageStartedEvent,
         ToolUsageFinishedEvent,
+        KnowledgeRetrievalStartedEvent,
+        KnowledgeRetrievalCompletedEvent,
     )
 
     crewai_installed = True
@@ -69,6 +71,14 @@ class CrewAIEventsListener(BaseEventListener):
 
         return execution_id
 
+    @staticmethod
+    def get_knowledge_execution_id(source, event) -> str:
+        source_id = id(source)
+        agent_id = id(event.agent) if hasattr(event, "agent") else "unknown"
+        execution_id = f"_knowledge_{source_id}_{agent_id}"
+
+        return execution_id
+
     def setup_listeners(self, crewai_event_bus):
         @crewai_event_bus.on(CrewKickoffStartedEvent)
         def on_crew_started(source, event: CrewKickoffStartedEvent):
@@ -161,6 +171,32 @@ class CrewAIEventsListener(BaseEventListener):
                     current_span.output = event.output
                 observer.__exit__(None, None, None)
 
+        @crewai_event_bus.on(KnowledgeRetrievalStartedEvent)
+        def on_knowledge_started(source, event: KnowledgeRetrievalStartedEvent):
+            observer = Observer(
+                span_type="tool",
+                func_name="knowledge_retrieval",
+                function_kwargs={},
+            )
+            self.span_observers[
+                self.get_knowledge_execution_id(source, event)
+            ] = observer
+            observer.__enter__()
+
+        @crewai_event_bus.on(KnowledgeRetrievalCompletedEvent)
+        def on_knowledge_completed(
+            source, event: KnowledgeRetrievalCompletedEvent
+        ):
+            observer = self.span_observers.pop(
+                self.get_knowledge_execution_id(source, event)
+            )
+            if observer:
+                current_span = current_span_context.get()
+                if current_span:
+                    current_span.input = event.query
+                    current_span.output = event.retrieved_knowledge
+                observer.__exit__(None, None, None)
+
 
 def instrument_crewai(api_key: Optional[str] = None):
     is_crewai_installed()