deepeval 3.4.8__py3-none-any.whl → 3.5.0__py3-none-any.whl

deepeval/config/settings_manager.py

@@ -0,0 +1,133 @@
+"""
+Applies CLI driven updates to the live Settings and optionally persists them to a
+dotenv file. Also syncs os.environ, handles unsets, and warns on unknown fields.
+Primary entrypoint: update_settings_and_persist.
+"""
+
+import logging
+import os
+
+from difflib import get_close_matches
+from pathlib import Path
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
+from enum import Enum
+
+from pydantic import SecretStr
+from deepeval.config.settings import get_settings, _SAVE_RE
+from deepeval.cli.dotenv_handler import DotenvHandler
+from deepeval.utils import bool_to_env_str
+
+logger = logging.getLogger(__name__)
+StrOrEnum = Union[str, Enum]
+
+
+def _env_key(k: StrOrEnum) -> str:
+    return k.value if isinstance(k, Enum) else str(k)
+
+
+def _normalize_for_env(val: Any) -> Optional[str]:
+    """Convert typed value to string for dotenv + os.environ; None -> unset."""
+    if val is None:
+        return None
+    if isinstance(val, SecretStr):
+        return val.get_secret_value()
+    if isinstance(val, bool):
+        return bool_to_env_str(val)
+    return str(val)
+
+
+def _resolve_save_path(save_opt: Optional[str]) -> Tuple[bool, Optional[Path]]:
+    """
+    Returns (ok, path).
+      - ok=False -> invalid save option format
+      - ok=True, path=None -> no persistence requested
+      - ok=True, path=Path -> persist to that file
+    """
+    raw = (
+        save_opt if save_opt is not None else os.getenv("DEEPEVAL_DEFAULT_SAVE")
+    )
+    if not raw:
+        return True, None
+    m = _SAVE_RE.match(raw.strip())
+    if not m:
+        return False, None
+    path = m.group("path") or ".env.local"
+    path = Path(os.path.expanduser(os.path.expandvars(path)))
+    return True, path
+
+
+def update_settings_and_persist(
+    updates: Mapping[StrOrEnum, Any],
+    *,
+    save: Optional[str] = None,
+    unset: Iterable[StrOrEnum] = (),
+    persist_dotenv: bool = True,
+) -> Tuple[bool, Optional[Path]]:
+    """
+    Write and update:
+      - validate + assign into live Settings()
+      - update os.environ
+      - persist to dotenv, if `save` or DEEPEVAL_DEFAULT_SAVE provided
+      - unset keys where value is None or explicitly in `unset`
+    Returns (handled, path_to_dotenv_if_any).
+    """
+    settings = get_settings()
+
+    # validate + assign into settings.
+    # validation is handled in Settings as long as validate_assignment=True
+    typed: Dict[str, Any] = {}
+    for key, value in updates.items():
+        k = _env_key(key)
+        if k not in type(settings).model_fields:
+            suggestion = get_close_matches(
+                k, type(settings).model_fields.keys(), n=1
+            )
+            if suggestion:
+                logger.warning(
+                    "Unknown settings field '%s'; did you mean '%s'? Ignoring.",
+                    k,
+                    suggestion[0],
+                    stacklevel=2,
+                )
+            else:
+                logger.warning(
+                    "Unknown settings field '%s'; ignoring.", k, stacklevel=2
+                )
+            continue
+
+        setattr(settings, k, value)
+        # coercion is handled in Settings
+        typed[k] = getattr(settings, k)
+
+    # build env maps
+    to_write: Dict[str, str] = {}
+    to_unset: set[str] = set(_env_key(k) for k in unset)
+
+    for k, v in typed.items():
+        env_val = _normalize_for_env(v)
+        if env_val is None:
+            to_unset.add(k)
+        else:
+            to_write[k] = env_val
+
+    # update process env so that it is effective immediately
+    for k, v in to_write.items():
+        os.environ[k] = v
+    for k in to_unset:
+        os.environ.pop(k, None)
+
+    if not persist_dotenv:
+        return True, None
+
+    # persist to dotenv if save is ok
+    ok, path = _resolve_save_path(save)
+    if not ok:
+        return False, None  # unsupported --save
+    if path:
+        h = DotenvHandler(path)
+        if to_write:
+            h.upsert(to_write)
+        if to_unset:
+            h.unset(to_unset)
+        return True, path
+    return True, None

deepeval/config/utils.py

@@ -0,0 +1,86 @@
+import os
+from typing import Any, Optional
+
+_TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on", "enable", "enabled"})
+_FALSY = frozenset({"0", "false", "f", "no", "n", "off", "disable", "disabled"})
+
+
+def parse_bool(value: Any, default: bool = False) -> bool:
+    """
+    Parse an arbitrary value into a boolean using env style semantics.
+
+    Truthy tokens (case-insensitive, quotes/whitespace ignored):
+      1, true, t, yes, y, on, enable, enabled
+    Falsy tokens:
+      0, false, f, no, n, off, disable, disabled
+
+    - bool -> returned as is
+    - None -> returns `default`
+    - int/float -> False if == 0, else True
+    - str/other -> matched against tokens above; non-matching -> `default`
+
+    Args:
+        value: Value to interpret.
+        default: Value to return if `value` is None or doesn’t match any token.
+
+    Returns:
+        The interpreted boolean.
+    """
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return default
+    if isinstance(value, (int, float)):
+        return value != 0
+
+    s = str(value).strip().strip('"').strip("'").lower()
+    if not s:
+        return default
+    if s in _TRUTHY:
+        return True
+    if s in _FALSY:
+        return False
+    return default
+
+
+def get_env_bool(key: str, default: bool = False) -> bool:
+    """
+    Read an environment variable and parse it as a boolean using `parse_bool`.
+
+    Args:
+        key: Environment variable name.
+        default: Returned when the variable is unset or does not match any token.
+
+    Returns:
+        Parsed boolean value.
+    """
+    return parse_bool(os.getenv(key), default)
+
+
+def bool_to_env_str(value: bool) -> str:
+    """
+    Canonicalize a boolean to the env/dotenv string form: "1" or "0".
+
+    Args:
+        value: Boolean to serialize.
+
+    Returns:
+        "1" if True, "0" if False.
+    """
+    return "1" if bool(value) else "0"
+
+
+def set_env_bool(key: str, value: Optional[bool] = False) -> None:
+    """
+    Set an environment variable to a canonical boolean string ("1" or "0").
+
+    Args:
+        key: The environment variable name to set.
+        value: The boolean value to store. If None, it is treated as False.
+               True -> "1", False/None -> "0".
+
+    Notes:
+        - This function always overwrites the variable in `os.environ`.
+        - Use `get_env_bool` to read back and parse the value safely.
+    """
+    os.environ[key] = bool_to_env_str(bool(value))

deepeval/dataset/__init__.py

@@ -1,4 +1,5 @@
 from .dataset import EvaluationDataset
 from .golden import Golden, ConversationalGolden
+from .test_run_tracer import init_global_test_run_tracer
 
 __all__ = ["EvaluationDataset", "Golden", "ConversationalGolden"]

deepeval/dataset/dataset.py

@@ -1,6 +1,8 @@
 from asyncio import Task
 from typing import Iterator, List, Optional, Union, Literal
 from dataclasses import dataclass, field
+from opentelemetry.trace import Tracer
+from opentelemetry.context import Context, attach, detach
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 import json
@@ -10,6 +12,8 @@ import os
 import datetime
 import time
 import ast
+import uuid
+from opentelemetry import baggage
 
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.dataset.utils import (
@@ -18,6 +22,7 @@ from deepeval.dataset.utils import (
     convert_convo_goldens_to_convo_test_cases,
     convert_convo_test_cases_to_convo_goldens,
     format_turns,
+    check_tracer,
     parse_turns,
     trimAndLoadJson,
 )
@@ -47,6 +52,7 @@ from deepeval.test_run import (
 from deepeval.dataset.types import global_evaluation_tasks
 from deepeval.openai.utils import openai_test_case_pairs
 from deepeval.tracing import trace_manager
+from deepeval.tracing.tracing import EVAL_DUMMY_SPAN_NAME
 
 
 valid_file_types = ["csv", "json", "jsonl"]
@@ -1097,6 +1103,7 @@ class EvaluationDataset:
         cache_config: Optional["CacheConfig"] = None,
         error_config: Optional["ErrorConfig"] = None,
         async_config: Optional["AsyncConfig"] = None,
+        run_otel: Optional[bool] = False,
     ) -> Iterator[Golden]:
         from deepeval.evaluate.utils import (
             aggregate_metric_pass_rates,
@@ -1133,9 +1140,14 @@ class EvaluationDataset:
             start_time = time.perf_counter()
             test_results: List[TestResult] = []
 
+            # sandwich start trace for OTEL
+            if run_otel:
+                ctx = self._start_otel_test_run()  # ignored span
+                ctx_token = attach(ctx)
+
             if async_config.run_async:
                 loop = get_or_create_event_loop()
-                yield from a_execute_agentic_test_cases_from_loop(
+                for golden in a_execute_agentic_test_cases_from_loop(
                     goldens=goldens,
                     identifier=identifier,
                     loop=loop,
@@ -1145,9 +1157,19 @@ class EvaluationDataset:
                     cache_config=cache_config,
                     error_config=error_config,
                     async_config=async_config,
-                )
+                ):
+                    if run_otel:
+                        _tracer = check_tracer()
+                        with _tracer.start_as_current_span(
+                            name=EVAL_DUMMY_SPAN_NAME,
+                            context=ctx,
+                        ):
+                            yield golden
+                    else:
+                        yield golden
+
             else:
-                yield from execute_agentic_test_cases_from_loop(
+                for golden in execute_agentic_test_cases_from_loop(
                     goldens=goldens,
                     trace_metrics=metrics,
                     display_config=display_config,
@@ -1155,7 +1177,16 @@ class EvaluationDataset:
                     error_config=error_config,
                     test_results=test_results,
                     identifier=identifier,
-                )
+                ):
+                    if run_otel:
+                        _tracer = check_tracer()
+                        with _tracer.start_as_current_span(
+                            name=EVAL_DUMMY_SPAN_NAME,
+                            context=ctx,
+                        ):
+                            yield golden
+                    else:
+                        yield golden
 
             end_time = time.perf_counter()
             run_duration = end_time - start_time
@@ -1184,12 +1215,41 @@ class EvaluationDataset:
             # clean up
             openai_test_case_pairs.clear()
             global_test_run_manager.save_test_run(TEMP_FILE_PATH)
-            confident_link = global_test_run_manager.wrap_up_test_run(
-                run_duration, display_table=False
-            )
-            return EvaluationResult(
-                test_results=test_results, confident_link=confident_link
-            )
+
+            # sandwich end trace for OTEL
+            if run_otel:
+                self._end_otel_test_run(ctx)
+                detach(ctx_token)
+
+            else:
+                confident_link = global_test_run_manager.wrap_up_test_run(
+                    run_duration, display_table=False
+                )
+                return EvaluationResult(
+                    test_results=test_results, confident_link=confident_link
+                )
 
     def evaluate(self, task: Task):
         global_evaluation_tasks.append(task)
+
+    def _start_otel_test_run(self, tracer: Optional[Tracer] = None) -> Context:
+        _tracer = check_tracer(tracer)
+        run_id = str(uuid.uuid4())
+        print("Starting OTLP test run with run_id: ", run_id)
+        ctx = baggage.set_baggage(
+            "confident.test_run.id", run_id, context=Context()
+        )
+        with _tracer.start_as_current_span(
+            "start_otel_test_run", context=ctx
+        ) as span:
+            span.set_attribute("confident.test_run.id", run_id)
+        return ctx
+
+    def _end_otel_test_run(self, ctx: Context, tracer: Optional[Tracer] = None):
+        run_id = baggage.get_baggage("confident.test_run.id", context=ctx)
+        print("Ending OTLP test run with run_id: ", run_id)
+        _tracer = check_tracer(tracer)
+        with _tracer.start_as_current_span(
+            "stop_otel_test_run", context=ctx
+        ) as span:
+            span.set_attribute("confident.test_run.id", run_id)

deepeval/dataset/test_run_tracer.py

@@ -0,0 +1,82 @@
+import os
+from typing import Optional
+from opentelemetry import baggage
+from opentelemetry.trace import Tracer as OTelTracer
+from opentelemetry.sdk.trace import SpanProcessor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+try:
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+
+    is_opentelemetry_installed = True
+except Exception:
+    is_opentelemetry_installed = False
+
+
+def is_opentelemetry_available():
+    if not is_opentelemetry_installed:
+        raise ImportError(
+            "OpenTelemetry SDK is not available. Please install it with `pip install opentelemetry-exporter-otlp-proto-http`."
+        )
+    return True
+
+
+from deepeval.confident.api import get_confident_api_key
+
+OTLP_ENDPOINT = (
+    os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+    else "https://otel.confident-ai.com"
+)
+# OTLP_ENDPOINT = "http://127.0.0.1:4318"
+
+# Module-level globals to be imported and used by other code
+GLOBAL_TEST_RUN_TRACER_PROVIDER: Optional[TracerProvider] = None
+GLOBAL_TEST_RUN_TRACER: Optional[OTelTracer] = None
+
+
+class RunIdSpanProcessor(SpanProcessor):
+    def on_start(self, span, parent_context):
+        run_id = baggage.get_baggage(
+            "confident.test_run.id", context=parent_context
+        )
+        if run_id:
+            span.set_attribute("confident.test_run.id", run_id)
+
+    def on_end(self, span) -> None:  # type: ignore[override]
+        # No-op
+        return None
+
+    def shutdown(self) -> None:  # type: ignore[override]
+        # No-op
+        return None
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:  # type: ignore[override]
+        # No-op
+        return True
+
+
+def init_global_test_run_tracer(api_key: Optional[str] = None):
+    is_opentelemetry_available()
+    api_key = get_confident_api_key()
+    if api_key is None:
+        raise ValueError("CONFIDENT_API_KEY is not set")
+
+    provider = TracerProvider()
+    exporter = OTLPSpanExporter(
+        endpoint=f"{OTLP_ENDPOINT}/v1/traces",
+        headers={"x-confident-api-key": api_key},
+    )
+    provider.add_span_processor(RunIdSpanProcessor())
+    provider.add_span_processor(BatchSpanProcessor(span_exporter=exporter))
+    tracer = provider.get_tracer("deepeval_tracer")
+
+    global GLOBAL_TEST_RUN_TRACER_PROVIDER
+    global GLOBAL_TEST_RUN_TRACER
+    GLOBAL_TEST_RUN_TRACER_PROVIDER = provider
+    GLOBAL_TEST_RUN_TRACER = tracer
+
+    return provider, tracer

deepeval/dataset/utils.py

@@ -2,6 +2,10 @@ from typing import List, Optional, Any
 import json
 import re
 
+from opentelemetry.trace import Tracer
+from opentelemetry import trace
+from opentelemetry.trace import NoOpTracerProvider
+
 from deepeval.dataset.api import Golden
 from deepeval.dataset.golden import ConversationalGolden
 from deepeval.test_case import LLMTestCase, ConversationalTestCase, Turn
@@ -151,3 +155,22 @@ def parse_turns(turns_str: str) -> List[Turn]:
             )
         )
     return res
+
+
+def check_tracer(tracer: Optional[Tracer] = None) -> Tracer:
+    if tracer:
+        return tracer
+    # Prefer module-level test-run tracer if available
+    try:
+        from deepeval.dataset.test_run_tracer import (
+            GLOBAL_TEST_RUN_TRACER,
+        )
+
+        if GLOBAL_TEST_RUN_TRACER is not None:
+            return GLOBAL_TEST_RUN_TRACER
+    except Exception:
+        raise RuntimeError(
+            "No global OpenTelemetry tracer provider is configured."  # TODO: link to docs
+        )
+
+    return GLOBAL_TEST_RUN_TRACER

deepeval/integrations/pydantic_ai/__init__.py

@@ -1,5 +1,3 @@
-from .agent import PydanticAIAgent as Agent
-from .setup import instrument_pydantic_ai
+from .patcher import instrument as instrument_pydantic_ai
 
-
-__all__ = ["Agent", "instrument_pydantic_ai"]
+__all__ = ["instrument_pydantic_ai"]

deepeval/integrations/pydantic_ai/{setup.py → otel.py}

@@ -31,14 +31,6 @@ def instrument_pydantic_ai(api_key: Optional[str] = None):
     with capture_tracing_integration("pydantic_ai"):
         is_opentelemetry_available()
 
-        if api_key:
-            deepeval.login(api_key)
-
-        api_key = get_confident_api_key()
-
-        if not api_key:
-            raise ValueError("No api key provided.")
-
         # create a new tracer provider
         tracer_provider = TracerProvider()
         tracer_provider.add_span_processor(