deepeval 3.5.4__py3-none-any.whl → 3.5.6__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.5.4"
+__version__: str = "3.5.6"

deepeval/cli/main.py

@@ -28,8 +28,6 @@ import typer
 from enum import Enum
 from pydantic import SecretStr
 from deepeval.key_handler import (
-    KEY_FILE_HANDLER,
-    KeyValues,
     EmbeddingKeyValues,
     ModelKeyValues,
 )
@@ -46,16 +44,9 @@ from deepeval.cli.utils import (
     render_login_message,
     upload_and_open_link,
     PROD,
-    resolve_save_target,
-    save_environ_to_store,
-    unset_environ_in_store,
-    switch_model_provider,
 )
 from deepeval.confident.api import (
-    get_confident_api_key,
     is_confident,
-    set_confident_api_key,
-    CONFIDENT_API_KEY_ENV_VAR,
 )
 
 app = typer.Typer(name="deepeval")
@@ -109,7 +100,7 @@ def set_confident_region_command(
     # Add flag emojis based on region
     flag = "🇺🇸" if region == Regions.US else "🇪🇺"
 
-    setting = get_settings()
+    settings = get_settings()
     with settings.edit(save=save) as edit_ctx:
         settings.CONFIDENT_REGION = region.value
 
@@ -282,23 +273,196 @@ def view():
             upload_and_open_link(_span=span)
 
 
-@app.command(name="enable-grpc-logging")
-def enable_grpc_logging(save: Optional[str] = None):
+@app.command(name="set-debug")
+def set_debug(
+    # Core verbosity
+    log_level: Optional[str] = typer.Option(
+        None,
+        "--log-level",
+        help="Global LOG_LEVEL (DEBUG|INFO|WARNING|ERROR|CRITICAL|NOTSET).",
+    ),
+    verbose: Optional[bool] = typer.Option(
+        None, "--verbose/--no-verbose", help="Toggle DEEPEVAL_VERBOSE_MODE."
+    ),
+    # Retry logging dials
+    retry_before_level: Optional[str] = typer.Option(
+        None,
+        "--retry-before-level",
+        help="Log level before a retry attempt (DEBUG|INFO|WARNING|ERROR|CRITICAL|NOTSET or numeric).",
+    ),
+    retry_after_level: Optional[str] = typer.Option(
+        None,
+        "--retry-after-level",
+        help="Log level after a retry attempt (DEBUG|INFO|WARNING|ERROR|CRITICAL|NOTSET or numeric).",
+    ),
+    # gRPC visibility
+    grpc: Optional[bool] = typer.Option(
+        None, "--grpc/--no-grpc", help="Toggle DEEPEVAL_GRPC_LOGGING."
+    ),
+    grpc_verbosity: Optional[str] = typer.Option(
+        None,
+        "--grpc-verbosity",
+        help="Set GRPC_VERBOSITY (DEBUG|INFO|ERROR|NONE).",
+    ),
+    grpc_trace: Optional[str] = typer.Option(
+        None,
+        "--grpc-trace",
+        help=(
+            "Set GRPC_TRACE to comma-separated tracer names or glob patterns "
+            "(e.g. 'tcp,http,secure_endpoint', '*' for all, 'list_tracers' to print available)."
+        ),
+    ),
+    # Confident tracing
+    trace_verbose: Optional[bool] = typer.Option(
+        None,
+        "--trace-verbose/--no-trace-verbose",
+        help="Enable / disable CONFIDENT_TRACE_VERBOSE.",
+    ),
+    trace_env: Optional[str] = typer.Option(
+        None,
+        "--trace-env",
+        help='Set CONFIDENT_TRACE_ENVIRONMENT ("development", "staging", "production", etc).',
+    ),
+    trace_flush: Optional[bool] = typer.Option(
+        None,
+        "--trace-flush/--no-trace-flush",
+        help="Enable / disable  CONFIDENT_TRACE_FLUSH.",
+    ),
+    # Advanced / potentially surprising
+    error_reporting: Optional[bool] = typer.Option(
+        None,
+        "--error-reporting/--no-error-reporting",
+        help="Enable / disable ERROR_REPORTING.",
+    ),
+    ignore_errors: Optional[bool] = typer.Option(
+        None,
+        "--ignore-errors/--no-ignore-errors",
+        help="Enable / disable IGNORE_DEEPEVAL_ERRORS (not recommended in normal debugging).",
+    ),
+    # Persistence
+    save: Optional[str] = typer.Option(
+        None,
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+):
     """
-    Enable verbose gRPC logging for the current process.
-    Pass --save=dotenv[:path] to persist it (optional).
+    Configure verbose debug behavior for DeepEval.
+
+    This command lets you mix-and-match verbosity flags (global LOG_LEVEL, verbose mode),
+    retry logger levels, gRPC wire logging, and Confident trace toggles. Values apply
+    immediately to the current process and can be persisted to a dotenv file with --save.
+
+    Examples:
+        deepeval set-debug --log-level DEBUG --verbose --grpc --retry-before-level DEBUG --retry-after-level INFO
+        deepeval set-debug --trace-verbose --trace-env staging --save dotenv:.env.local
     """
     settings = get_settings()
     with settings.edit(save=save) as edit_ctx:
-        settings.DEEPEVAL_GRPC_LOGGING = True
+        # Core verbosity
+        if log_level is not None:
+            settings.LOG_LEVEL = log_level
+        if verbose is not None:
+            settings.DEEPEVAL_VERBOSE_MODE = verbose
+
+        # Retry logging
+        if retry_before_level is not None:
+            settings.DEEPEVAL_RETRY_BEFORE_LOG_LEVEL = retry_before_level
+        if retry_after_level is not None:
+            settings.DEEPEVAL_RETRY_AFTER_LOG_LEVEL = retry_after_level
+
+        # gRPC
+        if grpc is not None:
+            settings.DEEPEVAL_GRPC_LOGGING = grpc
+        if grpc_verbosity is not None:
+            settings.GRPC_VERBOSITY = grpc_verbosity
+        if grpc_trace is not None:
+            settings.GRPC_TRACE = grpc_trace
+
+        # Confident tracing
+        if trace_verbose is not None:
+            settings.CONFIDENT_TRACE_VERBOSE = trace_verbose
+        if trace_env is not None:
+            settings.CONFIDENT_TRACE_ENVIRONMENT = trace_env
+        if trace_flush is not None:
+            settings.CONFIDENT_TRACE_FLUSH = trace_flush
+
+        # Advanced
+        if error_reporting is not None:
+            settings.ERROR_REPORTING = error_reporting
+        if ignore_errors is not None:
+            settings.IGNORE_DEEPEVAL_ERRORS = ignore_errors
+
+    handled, path, updated = edit_ctx.result
+
+    if not updated:
+        # no changes were made, so there is nothing to do.
+        return
+
+    if not handled and save is not None:
+        print("Unsupported --save option. Use --save=dotenv[:path].")
+    elif path:
+        print(
+            f"Saved environment variables to {path} (ensure it's git-ignored)."
+        )
+    else:
+        print(
+            "Settings updated for this session. To persist, use --save=dotenv[:path] "
+            "(default .env.local) or set DEEPEVAL_DEFAULT_SAVE=dotenv:.env.local"
+        )
+
+    print(":loud_sound: Debug options updated.")
+
+
+@app.command(name="unset-debug")
+def unset_debug(
+    save: Optional[str] = typer.Option(
+        None,
+        "--save",
+        help="Remove only the debug-related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+):
+    """
+    Restore default behavior by unsetting debug related variables.
+
+    Behavior:
+    - Resets LOG_LEVEL back to 'info'.
+    - Unsets DEEPEVAL_VERBOSE_MODE, retry log-level overrides, gRPC and Confident trace flags.
+    - If --save is provided (or DEEPEVAL_DEFAULT_SAVE is set), removes these keys from the target dotenv file.
+    """
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        # Back to normal global level
+        settings.LOG_LEVEL = "info"
+        settings.CONFIDENT_TRACE_ENVIRONMENT = "development"
+        settings.CONFIDENT_TRACE_VERBOSE = True
+
+        # Clear optional toggles/overrides
+        settings.DEEPEVAL_VERBOSE_MODE = None
+        settings.DEEPEVAL_RETRY_BEFORE_LOG_LEVEL = None
+        settings.DEEPEVAL_RETRY_AFTER_LOG_LEVEL = None
+
+        settings.DEEPEVAL_GRPC_LOGGING = None
+        settings.GRPC_VERBOSITY = None
+        settings.GRPC_TRACE = None
+
+        settings.CONFIDENT_TRACE_FLUSH = None
+
+        settings.ERROR_REPORTING = None
+        settings.IGNORE_DEEPEVAL_ERRORS = None
 
     handled, path, _ = edit_ctx.result
 
     if not handled and save is not None:
-        # invalid --save format (unsupported)
         print("Unsupported --save option. Use --save=dotenv[:path].")
+    elif path:
+        print(f"Removed debug-related environment variables from {path}.")
     else:
-        print("gRPC logging enabled.")
+        print("Debug settings reverted to defaults for this session.")
+
+    print(":mute: Debug options unset.")
 
 
 #############################################
@@ -1336,7 +1500,7 @@ def set_gemini_model_env(
         )
     else:
         print(
-            f":raising_hands: Congratulations! You're now using Gemini's model for all evals that require an LLM."
+            ":raising_hands: Congratulations! You're now using Gemini's model for all evals that require an LLM."
         )
 
 

deepeval/config/settings.py

@@ -281,6 +281,7 @@ class Settings(BaseSettings):
     #
     # Telemetry and Debug
     #
+    DEEPEVAL_DEBUG_ASYNC: Optional[bool] = None
     DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = None
     DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = None
     DEEPEVAL_GRPC_LOGGING: Optional[bool] = None
@@ -303,6 +304,19 @@ class Settings(BaseSettings):
     MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05
     MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0
 
+    #
+    # Async Task Configuration
+    #
+
+    # Maximum time allowed for a single task to complete
+    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS: int = (
+        300  # Set to float('inf') to disable timeout
+    )
+
+    # Buffer time for gathering results from all tasks, added to the longest task duration
+    # Increase if many tasks are running concurrently
+    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: int = 60
+
     ##############
     # Validators #
     ##############

deepeval/constants.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Union
 
 KEY_FILE: str = ".deepeval"
 HIDDEN_DIR: str = ".deepeval"
@@ -29,7 +30,7 @@ class ProviderSlug(str, Enum):
     OLLAMA = "ollama"
 
 
-def slugify(value: str | ProviderSlug) -> str:
+def slugify(value: Union[str, ProviderSlug]) -> str:
     return (
         value.value
         if isinstance(value, ProviderSlug)

deepeval/dataset/dataset.py

@@ -1,5 +1,5 @@
 from asyncio import Task
-from typing import Iterator, List, Optional, Union, Literal
+from typing import TYPE_CHECKING, Iterator, List, Optional, Union, Literal
 from dataclasses import dataclass, field
 from opentelemetry.trace import Tracer
 from opentelemetry.context import Context, attach, detach
@@ -7,7 +7,6 @@ from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 import json
 import csv
-import webbrowser
 import os
 import datetime
 import time
@@ -17,6 +16,7 @@ from opentelemetry import baggage
 
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.dataset.utils import (
+    coerce_to_task,
     convert_test_cases_to_goldens,
     convert_goldens_to_test_cases,
     convert_convo_goldens_to_convo_test_cases,
@@ -49,11 +49,18 @@ from deepeval.utils import (
 from deepeval.test_run import (
     global_test_run_manager,
 )
-from deepeval.dataset.types import global_evaluation_tasks
 from deepeval.openai.utils import openai_test_case_pairs
 from deepeval.tracing import trace_manager
 from deepeval.tracing.tracing import EVAL_DUMMY_SPAN_NAME
 
+if TYPE_CHECKING:
+    from deepeval.evaluate.configs import (
+        AsyncConfig,
+        DisplayConfig,
+        CacheConfig,
+        ErrorConfig,
+    )
+
 
 valid_file_types = ["csv", "json", "jsonl"]
 
@@ -1230,7 +1237,7 @@ class EvaluationDataset:
                 )
 
     def evaluate(self, task: Task):
-        global_evaluation_tasks.append(task)
+        coerce_to_task(task)
 
     def _start_otel_test_run(self, tracer: Optional[Tracer] = None) -> Context:
         _tracer = check_tracer(tracer)

deepeval/dataset/types.py

@@ -1,17 +1,25 @@
-class EvaluationTasks:
-    tasks: list = []
+import asyncio
 
-    def append(self, t):
-        self.tasks.append(t)
+from typing import Any
+from deepeval.dataset.utils import coerce_to_task
 
-    def get_tasks(self):
-        return self.tasks
 
-    def num_tasks(self):
-        return len(self.tasks)
+class EvaluationTasks:
 
-    def clear_tasks(self):
-        self.tasks.clear()
+    def __init__(self):
+        self._tasks: list[asyncio.Future] = []
 
+    def append(self, obj: Any):
+        self._tasks.append(coerce_to_task(obj))
+
+    def get_tasks(self) -> list[asyncio.Future]:
+        return list(self._tasks)
+
+    def num_tasks(self):
+        return len(self._tasks)
 
-global_evaluation_tasks = EvaluationTasks()
+    def clear_tasks(self) -> None:
+        for t in self._tasks:
+            if not t.done():
+                t.cancel()
+        self._tasks.clear()

deepeval/dataset/utils.py

@@ -1,10 +1,10 @@
-from typing import List, Optional, Any
+import asyncio
+import inspect
 import json
 import re
 
+from typing import List, Optional, Any
 from opentelemetry.trace import Tracer
-from opentelemetry import trace
-from opentelemetry.trace import NoOpTracerProvider
 
 from deepeval.dataset.api import Golden
 from deepeval.dataset.golden import ConversationalGolden
@@ -174,3 +174,31 @@ def check_tracer(tracer: Optional[Tracer] = None) -> Tracer:
         )
 
     return GLOBAL_TEST_RUN_TRACER
+
+
+def coerce_to_task(obj: Any) -> asyncio.Future[Any]:
+    # already a Task so just return it
+    if isinstance(obj, asyncio.Task):
+        return obj
+
+    # If it is a future, it is already scheduled, so just return it
+    if asyncio.isfuture(obj):
+        # type: ignore[return-value]  # it is an awaitable, gather accepts it
+        return obj
+
+    # bare coroutine must be explicitly scheduled using create_task to bind to loop & track
+    if asyncio.iscoroutine(obj):
+        return asyncio.create_task(obj)
+
+    # generic awaitable (any object with __await__) will need to be wrapped so create_task accepts it
+    if inspect.isawaitable(obj):
+
+        async def _wrap(awaitable):
+            return await awaitable
+
+        return asyncio.create_task(_wrap(obj))
+
+    # not awaitable, so time to sound the alarm!
+    raise TypeError(
+        f"Expected Task/Future/coroutine/awaitable, got {type(obj).__name__}"
+    )