deepeval 3.4.7__py3-none-any.whl → 3.4.9__py3-none-any.whl

deepeval/test_run/api.py

@@ -50,6 +50,7 @@ class LLMApiTestCase(BaseModel):
     trace: Optional[TraceApi] = Field(None)
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
+    # metric_collection: Optional[str] = Field(None, alias="metricCollection")
 
     def update_metric_data(self, metric_data: MetricData):
         if self.metrics_data is None:

deepeval/tracing/otel/exporter.py

@@ -30,6 +30,7 @@ from deepeval.tracing.otel.utils import (
     to_hex_string,
     parse_string,
     parse_list_of_strings,
+    post_test_run,
 )
 from deepeval.tracing import perf_epoch_bridge as peb
 from deepeval.tracing.types import TraceAttributes
@@ -80,7 +81,8 @@ class ConfidentSpanExporter(SpanExporter):
         self,
         spans: typing.Sequence[ReadableSpan],
         timeout_millis: int = 30000,
-        api_key: Optional[str] = None,  # dynamic api key
+        api_key: Optional[str] = None,  # dynamic api key,
+        _test_run_id: Optional[str] = None,
     ) -> SpanExportResult:
         # build forest of spans
         forest = self._build_span_forest(spans)
@@ -223,14 +225,24 @@ class ConfidentSpanExporter(SpanExporter):
                 trace_manager.add_span_to_trace(base_span_wrapper.base_span)
                 # no removing span because it can be parent of other spans
 
-        # safely end all active traces
+        # safely end all active traces or return them for test runs
         active_traces_keys = list(trace_manager.active_traces.keys())
-        for trace_key in active_traces_keys:
-            set_trace_time(trace_manager.get_trace_by_uuid(trace_key))
-            trace_manager.end_trace(trace_key)
-        trace_manager.clear_traces()
-
-        return SpanExportResult.SUCCESS
+        if _test_run_id:
+            traces = []
+            for trace_key in active_traces_keys:
+                set_trace_time(trace_manager.get_trace_by_uuid(trace_key))
+                trace = trace_manager.get_trace_by_uuid(trace_key)
+                if trace:
+                    traces.append(trace)
+            trace_manager.clear_traces()
+            post_test_run(traces, _test_run_id)
+            return SpanExportResult.SUCCESS
+        else:
+            for trace_key in active_traces_keys:
+                set_trace_time(trace_manager.get_trace_by_uuid(trace_key))
+                trace_manager.end_trace(trace_key)
+            trace_manager.clear_traces()
+            return SpanExportResult.SUCCESS
 
     def _convert_readable_span_to_base_span(
         self, span: ReadableSpan

deepeval/tracing/otel/utils.py

@@ -1,5 +1,6 @@
 from typing import List, Optional, Tuple, Any
 from deepeval.tracing.types import Trace, LLMTestCase, ToolCall
+from deepeval.tracing import trace_manager, BaseSpan
 from opentelemetry.sdk.trace.export import ReadableSpan
 import json
 
@@ -250,3 +251,59 @@ def parse_list_of_strings(context: List[str]) -> List[str]:
             else:
                 parsed_context.append(context_str)
     return parsed_context
+
+
+from deepeval.evaluate.utils import create_api_test_case
+from deepeval.test_run.api import LLMApiTestCase
+from deepeval.test_run.test_run import global_test_run_manager
+from typing import Optional
+
+
+def post_test_run(traces: List[Trace], test_run_id: Optional[str]):
+    # Accept single trace or list of traces
+    if isinstance(traces, Trace):
+        traces = [traces]
+
+    api_test_cases: List[LLMApiTestCase] = []
+
+    # Collect test cases from spans that have metric_collection
+    for trace in traces:
+        trace_api = trace_manager.create_trace_api(trace)
+
+        def dfs(span: BaseSpan):
+            if span.metric_collection:
+                llm_test_case = LLMTestCase(
+                    input=str(span.input),
+                    actual_output=(
+                        str(span.output) if span.output is not None else None
+                    ),
+                    expected_output=span.expected_output,
+                    context=span.context,
+                    retrieval_context=span.retrieval_context,
+                    tools_called=span.tools_called,
+                    expected_tools=span.expected_tools,
+                )
+                api_case = create_api_test_case(
+                    test_case=llm_test_case,
+                    trace=trace_api,
+                    index=None,
+                )
+                if isinstance(api_case, LLMApiTestCase):
+                    api_case.metric_collection = span.metric_collection
+                    api_test_cases.append(api_case)
+
+            for child in span.children or []:
+                dfs(child)
+
+        for root in trace.root_spans:
+            dfs(root)
+
+    # Prepare and post TestRun using the global test run manager
+    test_run_manager = global_test_run_manager
+    test_run_manager.create_test_run(identifier=test_run_id)
+    test_run = test_run_manager.get_test_run()
+
+    for case in api_test_cases:
+        test_run.add_test_case(case)
+
+    # return test_run_manager.post_test_run(test_run) TODO: add after test run with metric collection is implemented

deepeval/tracing/tracing.py

@@ -9,16 +9,16 @@ import atexit
 import queue
 import uuid
 import os
+import json
+import time
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
 
-
+from deepeval.config.settings import get_settings
 from deepeval.constants import (
     CONFIDENT_TRACE_VERBOSE,
     CONFIDENT_TRACE_FLUSH,
-    CONFIDENT_SAMPLE_RATE,
-    CONFIDENT_TRACE_ENVIRONMENT,
 )
 from deepeval.confident.api import Api, Endpoints, HttpMethods, is_confident
 from deepeval.metrics import BaseMetric
@@ -50,12 +50,16 @@ from deepeval.tracing.utils import (
     tracing_enabled,
     validate_environment,
     validate_sampling_rate,
+    dump_body_to_json_file,
+    get_deepeval_trace_mode,
 )
 from deepeval.utils import dataclass_to_dict
 from deepeval.tracing.context import current_span_context, current_trace_context
 from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.tracing.api import PromptApi
 
+EVAL_DUMMY_SPAN_NAME = "evals_iterator"
+
 
 class TraceManager:
     def __init__(self):
@@ -65,25 +69,27 @@ class TraceManager:
             {}
         )  # Map of span_uuid to BaseSpan
 
+        settings = get_settings()
         # Initialize queue and worker thread for trace posting
         self._trace_queue = queue.Queue()
         self._worker_thread = None
         self._min_interval = 0.2  # Minimum time between API calls (seconds)
         self._last_post_time = 0
         self._in_flight_tasks: Set[asyncio.Task[Any]] = set()
-        self._daemon = (
-            False if os.getenv(CONFIDENT_TRACE_FLUSH) == "YES" else True
-        )
+        self._flush_enabled = bool(settings.CONFIDENT_TRACE_FLUSH)
+        self._daemon = not self._flush_enabled
 
         # trace manager attributes
         self.confident_api_key = None
         self.custom_mask_fn: Optional[Callable] = None
-        self.environment = os.environ.get(
-            CONFIDENT_TRACE_ENVIRONMENT, Environment.DEVELOPMENT.value
+        self.environment = (
+            settings.CONFIDENT_TRACE_ENVIRONMENT
+            if settings.CONFIDENT_TRACE_ENVIRONMENT is not None
+            else Environment.DEVELOPMENT.value
         )
         validate_environment(self.environment)
 
-        self.sampling_rate = os.environ.get(CONFIDENT_SAMPLE_RATE, 1)
+        self.sampling_rate = settings.CONFIDENT_SAMPLE_RATE
         validate_sampling_rate(self.sampling_rate)
         self.openai_client = None
         self.tracing_enabled = True
@@ -103,7 +109,8 @@ class TraceManager:
         queue_size = self._trace_queue.qsize()
         in_flight = len(self._in_flight_tasks)
         remaining_tasks = queue_size + in_flight
-        if os.getenv(CONFIDENT_TRACE_FLUSH) != "YES" and remaining_tasks > 0:
+
+        if not self._flush_enabled and remaining_tasks > 0:
             self._print_trace_status(
                 message=f"WARNING: Exiting with {queue_size + in_flight} abaonded trace(s).",
                 trace_worker_status=TraceWorkerStatus.WARNING,
@@ -179,8 +186,14 @@ class TraceManager:
             if trace.status == TraceSpanStatus.IN_PROGRESS:
                 trace.status = TraceSpanStatus.SUCCESS
 
+            mode = get_deepeval_trace_mode()
+            if mode == "gen":
+                body = self.create_trace_api(trace).model_dump(
+                    by_alias=True, exclude_none=True
+                )
+                dump_body_to_json_file(body)
             # Post the trace to the server before removing it
-            if not self.evaluating:
+            elif not self.evaluating:
                 self.post_trace(trace)
             else:
                 if self.evaluation_loop:
@@ -237,6 +250,15 @@ class TraceManager:
             # This is a child span, find its parent and add it to the parent's children
             parent_span = self.get_span_by_uuid(span.parent_uuid)
             if parent_span:
+
+                if (
+                    parent_span.name == EVAL_DUMMY_SPAN_NAME
+                ):  # ignored span for evaluation
+                    span.parent_uuid = None
+                    trace.root_spans.remove(parent_span)
+                    trace.root_spans.append(span)
+                    return
+
                 parent_span.children.append(span)
             else:
                 trace.root_spans.append(span)
@@ -274,10 +296,7 @@ class TraceManager:
         description: Optional[str] = None,
         environment: Optional[str] = None,
     ):
-        if (
-            os.getenv(CONFIDENT_TRACE_VERBOSE, "YES").upper() != "NO"
-            and self.evaluating is False
-        ):
+        if get_settings().CONFIDENT_TRACE_VERBOSE and self.evaluating is False:
             console = Console()
             message_prefix = "[dim][Confident AI Trace Log][/dim]"
             if trace_worker_status == TraceWorkerStatus.SUCCESS:
@@ -401,6 +420,7 @@ class TraceManager:
                         api = Api(api_key=trace_api.confident_api_key)
                     else:
                         api = Api(api_key=self.confident_api_key)
+
                     api_response, link = await api.a_send_request(
                         method=HttpMethods.POST,
                         endpoint=Endpoints.TRACES_ENDPOINT,
@@ -415,7 +435,7 @@ class TraceManager:
                         description=link,
                         environment=self.environment,
                     )
-                elif os.getenv(CONFIDENT_TRACE_FLUSH) == "YES":
+                elif self._flush_enabled:
                     # Main thread gone → to be flushed
                     remaining_trace_request_bodies.append(body)
 
@@ -492,6 +512,7 @@ class TraceManager:
             with capture_send_trace():
                 try:
                     api = Api(api_key=self.confident_api_key)
+
                     _, link = api.send_request(
                         method=HttpMethods.POST,
                         endpoint=Endpoints.TRACES_ENDPOINT,

deepeval/tracing/utils.py

@@ -1,8 +1,15 @@
 import os
+import time
+import inspect
+import json
+import sys
+import difflib
 from datetime import datetime, timezone
 from enum import Enum
 from time import perf_counter
+import time
 from collections import deque
+from typing import Any, Dict, Optional, Sequence, Callable
 
 from deepeval.constants import CONFIDENT_TRACING_ENABLED
 
@@ -14,6 +21,12 @@ class Environment(Enum):
     TESTING = "testing"
 
 
+def _strip_nul(s: str) -> str:
+    # Replace embedded NUL, which Postgres cannot store in text/jsonb
+    # Do NOT try to escape as \u0000 because PG will still reject it.
+    return s.replace("\x00", "")
+
+
 def tracing_enabled():
     return os.getenv(CONFIDENT_TRACING_ENABLED, "YES").upper() == "YES"
 
@@ -42,6 +55,11 @@ def make_json_serializable(obj):
 
     def _serialize(o):
         oid = id(o)
+
+        # strip Nulls
+        if isinstance(o, str):
+            return _strip_nul(o)
+
         # Primitive types are already serializable
         if isinstance(o, (str, int, float, bool)) or o is None:
             return o
@@ -77,7 +95,7 @@ def make_json_serializable(obj):
             return result
 
         # Fallback: convert to string
-        return str(o)
+        return _strip_nul(str(o))
 
     return _serialize(obj)
 
@@ -115,3 +133,82 @@ def replace_self_with_class_name(obj):
         return f"<{obj.__class__.__name__}>"
     except:
         return f"<self>"
+
+
+def get_deepeval_trace_mode() -> Optional[str]:
+    deepeval_trace_mode = None
+    try:
+        args = sys.argv
+        for idx, arg in enumerate(args):
+            if isinstance(arg, str) and arg.startswith(
+                "--deepeval-trace-mode="
+            ):
+                deepeval_trace_mode = (
+                    arg.split("=", 1)[1].strip().strip('"').strip("'").lower()
+                )
+                break
+            if arg == "--deepeval-trace-mode" and idx + 1 < len(args):
+                deepeval_trace_mode = (
+                    str(args[idx + 1]).strip().strip('"').strip("'").lower()
+                )
+                break
+    except Exception:
+        deepeval_trace_mode = None
+
+    return deepeval_trace_mode
+
+
+def dump_body_to_json_file(
+    body: Dict[str, Any], file_path: Optional[str] = None
+) -> str:
+    entry_file = None
+    try:
+        cmd0 = sys.argv[0] if sys.argv else None
+        if cmd0 and cmd0.endswith(".py"):
+            entry_file = cmd0
+        else:
+            for frame_info in reversed(inspect.stack()):
+                fp = frame_info.filename
+                if (
+                    fp
+                    and fp.endswith(".py")
+                    and "deepeval/tracing" not in fp
+                    and "site-packages" not in fp
+                ):
+                    entry_file = fp
+                    break
+    except Exception:
+        entry_file = None
+
+    if not entry_file:
+        entry_file = "unknown.py"
+
+    abs_entry = os.path.abspath(entry_file)
+    dir_path = os.path.dirname(abs_entry)
+
+    file_arg = None
+    try:
+        for idx, arg in enumerate(sys.argv):
+            if isinstance(arg, str) and arg.startswith(
+                "--deepeval-trace-file-name="
+            ):
+                file_arg = arg.split("=", 1)[1].strip().strip('"').strip("'")
+                break
+            if arg == "--deepeval-trace-file-name" and idx + 1 < len(sys.argv):
+                file_arg = str(sys.argv[idx + 1]).strip().strip('"').strip("'")
+                break
+    except Exception:
+        file_arg = None
+
+    if file_path:
+        dst_path = os.path.abspath(file_path)
+    elif file_arg:
+        dst_path = os.path.abspath(file_arg)
+    else:
+        base_name = os.path.splitext(os.path.basename(abs_entry))[0]
+        dst_path = os.path.join(dir_path, f"{base_name}.json")
+
+    actual_body = make_json_serializable(body)
+    with open(dst_path, "w", encoding="utf-8") as f:
+        json.dump(actual_body, f, ensure_ascii=False, indent=2, sort_keys=True)
+    return dst_path

deepeval/utils.py

@@ -1,26 +1,34 @@
-from contextvars import ContextVar
-from enum import Enum
 import copy
 import os
 import json
 import time
-from typing import Any, Optional, Dict, List, Union
-from collections.abc import Iterable
 import webbrowser
 import tqdm
 import re
 import string
-from dataclasses import asdict, is_dataclass
-import re
 import asyncio
 import nest_asyncio
 import uuid
+import math
+
+from contextvars import ContextVar
+from enum import Enum
+from typing import Any, Optional, Dict, List, Union
+from collections.abc import Iterable
+from dataclasses import asdict, is_dataclass
 from pydantic import BaseModel
 from rich.progress import Progress
 from rich.console import Console, Theme
 
 from deepeval.confident.api import set_confident_api_key
 from deepeval.constants import CONFIDENT_OPEN_BROWSER
+from deepeval.config.settings import get_settings
+from deepeval.config.utils import (
+    parse_bool,
+    get_env_bool,
+    bool_to_env_str,
+    set_env_bool,
+)
 
 
 def get_lcs(seq1, seq2):
@@ -140,82 +148,55 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
     return loop
 
 
-def should_skip_on_missing_params():
-    try:
-        if os.environ["SKIP_DEEPEVAL_MISSING_PARAMS"] == "YES":
-            return True
-        else:
-            return False
-    except:
-        return False
+def set_should_skip_on_missing_params(yes: bool):
+    s = get_settings()
+    with s.edit(persist=False):
+        s.SKIP_DEEPEVAL_MISSING_PARAMS = yes
 
 
-def set_should_skip_on_missing_params(yes: bool):
-    if yes:
-        os.environ["SKIP_DEEPEVAL_MISSING_PARAMS"] = "YES"
-    else:
-        os.environ["SKIP_DEEPEVAL_MISSING_PARAMS"] = "NO"
+def should_ignore_errors() -> bool:
+    return bool(get_settings().IGNORE_DEEPEVAL_ERRORS)
 
 
-def should_ignore_errors():
-    try:
-        if os.environ["IGNORE_DEEPEVAL_ERRORS"] == "YES":
-            return True
-        else:
-            return False
-    except:
-        return False
+def should_skip_on_missing_params() -> bool:
+    return bool(get_settings().SKIP_DEEPEVAL_MISSING_PARAMS)
 
 
 def set_should_ignore_errors(yes: bool):
-    if yes:
-        os.environ["IGNORE_DEEPEVAL_ERRORS"] = "YES"
-    else:
-        os.environ["IGNORE_DEEPEVAL_ERRORS"] = "NO"
+    s = get_settings()
+    with s.edit(persist=False):
+        s.IGNORE_DEEPEVAL_ERRORS = yes
 
 
-def should_verbose_print() -> Union[bool, None]:
-    try:
-        if os.environ["DEEPEVAL_VERBOSE_MODE"] == "YES":
-            return True
-        else:
-            return None
-    except:
-        return None
+def should_verbose_print() -> bool:
+    return bool(get_settings().DEEPEVAL_VERBOSE_MODE)
 
 
 def set_verbose_mode(yes: Optional[bool]):
-    if yes:
-        os.environ["DEEPEVAL_VERBOSE_MODE"] = "YES"
+    s = get_settings()
+    with s.edit(persist=False):
+        s.DEEPEVAL_VERBOSE_MODE = yes
 
 
 def set_identifier(identifier: Optional[str]):
     if identifier:
-        os.environ["DEEPEVAL_IDENTIFIER"] = identifier
+        s = get_settings()
+        with s.edit(persist=False):
+            s.DEEPEVAL_IDENTIFIER = identifier
 
 
 def get_identifier() -> Optional[str]:
-    try:
-        return os.environ["DEEPEVAL_IDENTIFIER"]
-    except:
-        return None
+    return get_settings().DEEPEVAL_IDENTIFIER
 
 
-def should_use_cache():
-    try:
-        if os.environ["ENABLE_DEEPEVAL_CACHE"] == "YES":
-            return True
-        else:
-            return False
-    except:
-        return False
+def should_use_cache() -> bool:
+    return bool(get_settings().ENABLE_DEEPEVAL_CACHE)
 
 
 def set_should_use_cache(yes: bool):
-    if yes:
-        os.environ["ENABLE_DEEPEVAL_CACHE"] = "YES"
-    else:
-        os.environ["ENABLE_DEEPEVAL_CACHE"] = "NO"
+    s = get_settings()
+    with s.edit(persist=False):
+        s.ENABLE_DEEPEVAL_CACHE = yes
 
 
 def login(api_key: str):
@@ -233,17 +214,11 @@ def login(api_key: str):
 
 
 def set_is_running_deepeval(flag: bool):
-    if flag:
-        os.environ["DEEPEVAL"] = "YES"
-    else:
-        os.environ["DEEPEVAL"] = "NO"
+    set_env_bool("DEEPEVAL", flag)
 
 
 def get_is_running_deepeval() -> bool:
-    try:
-        return os.environ["DEEPEVAL"] == "YES"
-    except:
-        return False
+    return get_env_bool("DEEPEVAL")
 
 
 def is_in_ci_env() -> bool:
@@ -270,8 +245,8 @@ def is_in_ci_env() -> bool:
 
 
 def open_browser(url: str):
-    if os.getenv(CONFIDENT_OPEN_BROWSER) != "NO":
-        if is_in_ci_env() == False:
+    if get_settings().CONFIDENT_OPEN_BROWSER:
+        if not is_in_ci_env():
             webbrowser.open(url)
 
 
@@ -439,6 +414,8 @@ def normalize_text(text: str) -> str:
 
 
 def get_freer_gpu():
+    import numpy as np
+
     os.system("nvidia-smi -q -d Memory |grep -A4 GPU|grep Free >tmp_smi")
     memory_available = [
         int(x.split()[2]) + 5 * i
@@ -466,8 +443,11 @@ def wait_free_gpu(gb_needed):
 def select_freer_gpu():
     freer_gpu = str(get_freer_gpu())
     print("Will use GPU: %s" % (freer_gpu))
-    os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
-    os.environ["CUDA_VISIBLE_DEVICES"] = "" + freer_gpu
+
+    s = get_settings()
+    with s.edit(persist=False):
+        s.CUDA_LAUNCH_BLOCKING = True
+        s.CUDA_VISIBLE_DEVICES = freer_gpu
     return freer_gpu
 
 
@@ -535,6 +515,67 @@ def remove_pbars(
         progress.remove_task(pbar_id)
 
 
+def read_env_int(
+    name: str, default: int, *, min_value: int | None = None
+) -> int:
+    """Read an integer from an environment variable with safe fallback.
+
+    Attempts to read os.environ[name] and parse it as an int. If the variable
+    is unset, cannot be parsed, or is less than `min_value` (when provided),
+    the function returns `default`.
+
+    Args:
+        name: Environment variable name to read.
+        default: Value to return when the env var is missing/invalid/out of range.
+        min_value: Optional inclusive lower bound; values < min_value are rejected.
+
+    Returns:
+        The parsed integer, or `default` on any failure.
+    """
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    try:
+        v = int(raw)
+        if min_value is not None and v < min_value:
+            return default
+        return v
+    except Exception:
+        return default
+
+
+def read_env_float(
+    name: str, default: float, *, min_value: float | None = None
+) -> float:
+    """Read a float from an environment variable with safe fallback.
+
+    Attempts to read os.environ[name] and parse it as a float. If the variable
+    is unset, cannot be parsed, or is less than `min_value` (when provided),
+    the function returns `default`.
+
+    Args:
+        name: Environment variable name to read.
+        default: Value to return when the env var is missing/invalid/out of range.
+        min_value: Optional inclusive lower bound; values < min_value are rejected.
+
+    Returns:
+        The parsed float, or `default` on any failure.
+    """
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    try:
+        v = float(raw)
+    except Exception:
+        return default
+
+    if not math.isfinite(v):
+        return default
+    if min_value is not None and v < min_value:
+        return default
+    return v
+
+
 my_theme = Theme(
     {
         "bar.complete": "#11ff00",

{deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.4.7
+Version: 3.4.9
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -25,6 +25,8 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: portalocker
 Requires-Dist: posthog (>=6.3.0,<7.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.10.1,<3.0.0)
 Requires-Dist: pyfiglet
 Requires-Dist: pytest
 Requires-Dist: pytest-asyncio