deepeval 3.4.7__py3-none-any.whl → 3.4.9__py3-none-any.whl

deepeval/config/utils.py

@@ -0,0 +1,86 @@
+import os
+from typing import Any, Optional
+
+_TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on", "enable", "enabled"})
+_FALSY = frozenset({"0", "false", "f", "no", "n", "off", "disable", "disabled"})
+
+
+def parse_bool(value: Any, default: bool = False) -> bool:
+    """
+    Parse an arbitrary value into a boolean using env style semantics.
+
+    Truthy tokens (case-insensitive, quotes/whitespace ignored):
+      1, true, t, yes, y, on, enable, enabled
+    Falsy tokens:
+      0, false, f, no, n, off, disable, disabled
+
+    - bool -> returned as is
+    - None -> returns `default`
+    - int/float -> False if == 0, else True
+    - str/other -> matched against tokens above; non-matching -> `default`
+
+    Args:
+        value: Value to interpret.
+        default: Value to return if `value` is None or doesn’t match any token.
+
+    Returns:
+        The interpreted boolean.
+    """
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return default
+    if isinstance(value, (int, float)):
+        return value != 0
+
+    s = str(value).strip().strip('"').strip("'").lower()
+    if not s:
+        return default
+    if s in _TRUTHY:
+        return True
+    if s in _FALSY:
+        return False
+    return default
+
+
+def get_env_bool(key: str, default: bool = False) -> bool:
+    """
+    Read an environment variable and parse it as a boolean using `parse_bool`.
+
+    Args:
+        key: Environment variable name.
+        default: Returned when the variable is unset or does not match any token.
+
+    Returns:
+        Parsed boolean value.
+    """
+    return parse_bool(os.getenv(key), default)
+
+
+def bool_to_env_str(value: bool) -> str:
+    """
+    Canonicalize a boolean to the env/dotenv string form: "1" or "0".
+
+    Args:
+        value: Boolean to serialize.
+
+    Returns:
+        "1" if True, "0" if False.
+    """
+    return "1" if bool(value) else "0"
+
+
+def set_env_bool(key: str, value: Optional[bool] = False) -> None:
+    """
+    Set an environment variable to a canonical boolean string ("1" or "0").
+
+    Args:
+        key: The environment variable name to set.
+        value: The boolean value to store. If None, it is treated as False.
+               True -> "1", False/None -> "0".
+
+    Notes:
+        - This function always overwrites the variable in `os.environ`.
+        - Use `get_env_bool` to read back and parse the value safely.
+    """
+    os.environ[key] = bool_to_env_str(bool(value))

deepeval/dataset/__init__.py

@@ -1,4 +1,5 @@
 from .dataset import EvaluationDataset
 from .golden import Golden, ConversationalGolden
+from .test_run_tracer import init_global_test_run_tracer
 
 __all__ = ["EvaluationDataset", "Golden", "ConversationalGolden"]

deepeval/dataset/dataset.py

@@ -1,6 +1,8 @@
 from asyncio import Task
 from typing import Iterator, List, Optional, Union, Literal
 from dataclasses import dataclass, field
+from opentelemetry.trace import Tracer
+from opentelemetry.context import Context, attach, detach
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 import json
@@ -10,6 +12,8 @@ import os
 import datetime
 import time
 import ast
+import uuid
+from opentelemetry import baggage
 
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.dataset.utils import (
@@ -18,6 +22,7 @@ from deepeval.dataset.utils import (
     convert_convo_goldens_to_convo_test_cases,
     convert_convo_test_cases_to_convo_goldens,
     format_turns,
+    check_tracer,
     parse_turns,
     trimAndLoadJson,
 )
@@ -47,6 +52,7 @@ from deepeval.test_run import (
 from deepeval.dataset.types import global_evaluation_tasks
 from deepeval.openai.utils import openai_test_case_pairs
 from deepeval.tracing import trace_manager
+from deepeval.tracing.tracing import EVAL_DUMMY_SPAN_NAME
 
 
 valid_file_types = ["csv", "json", "jsonl"]
@@ -1097,6 +1103,7 @@ class EvaluationDataset:
         cache_config: Optional["CacheConfig"] = None,
         error_config: Optional["ErrorConfig"] = None,
         async_config: Optional["AsyncConfig"] = None,
+        run_otel: Optional[bool] = False,
     ) -> Iterator[Golden]:
         from deepeval.evaluate.utils import (
             aggregate_metric_pass_rates,
@@ -1133,9 +1140,14 @@ class EvaluationDataset:
             start_time = time.perf_counter()
             test_results: List[TestResult] = []
 
+            # sandwich start trace for OTEL
+            if run_otel:
+                ctx = self._start_otel_test_run()  # ignored span
+                ctx_token = attach(ctx)
+
             if async_config.run_async:
                 loop = get_or_create_event_loop()
-                yield from a_execute_agentic_test_cases_from_loop(
+                for golden in a_execute_agentic_test_cases_from_loop(
                     goldens=goldens,
                     identifier=identifier,
                     loop=loop,
@@ -1145,9 +1157,19 @@ class EvaluationDataset:
                     cache_config=cache_config,
                     error_config=error_config,
                     async_config=async_config,
-                )
+                ):
+                    if run_otel:
+                        _tracer = check_tracer()
+                        with _tracer.start_as_current_span(
+                            name=EVAL_DUMMY_SPAN_NAME,
+                            context=ctx,
+                        ):
+                            yield golden
+                    else:
+                        yield golden
+
             else:
-                yield from execute_agentic_test_cases_from_loop(
+                for golden in execute_agentic_test_cases_from_loop(
                     goldens=goldens,
                     trace_metrics=metrics,
                     display_config=display_config,
@@ -1155,7 +1177,16 @@ class EvaluationDataset:
                     error_config=error_config,
                     test_results=test_results,
                     identifier=identifier,
-                )
+                ):
+                    if run_otel:
+                        _tracer = check_tracer()
+                        with _tracer.start_as_current_span(
+                            name=EVAL_DUMMY_SPAN_NAME,
+                            context=ctx,
+                        ):
+                            yield golden
+                    else:
+                        yield golden
 
             end_time = time.perf_counter()
             run_duration = end_time - start_time
@@ -1184,12 +1215,41 @@ class EvaluationDataset:
             # clean up
             openai_test_case_pairs.clear()
             global_test_run_manager.save_test_run(TEMP_FILE_PATH)
-            confident_link = global_test_run_manager.wrap_up_test_run(
-                run_duration, display_table=False
-            )
-            return EvaluationResult(
-                test_results=test_results, confident_link=confident_link
-            )
+
+            # sandwich end trace for OTEL
+            if run_otel:
+                self._end_otel_test_run(ctx)
+                detach(ctx_token)
+
+            else:
+                confident_link = global_test_run_manager.wrap_up_test_run(
+                    run_duration, display_table=False
+                )
+                return EvaluationResult(
+                    test_results=test_results, confident_link=confident_link
+                )
 
     def evaluate(self, task: Task):
         global_evaluation_tasks.append(task)
+
+    def _start_otel_test_run(self, tracer: Optional[Tracer] = None) -> Context:
+        _tracer = check_tracer(tracer)
+        run_id = str(uuid.uuid4())
+        print("Starting OTLP test run with run_id: ", run_id)
+        ctx = baggage.set_baggage(
+            "confident.test_run.id", run_id, context=Context()
+        )
+        with _tracer.start_as_current_span(
+            "start_otel_test_run", context=ctx
+        ) as span:
+            span.set_attribute("confident.test_run.id", run_id)
+        return ctx
+
+    def _end_otel_test_run(self, ctx: Context, tracer: Optional[Tracer] = None):
+        run_id = baggage.get_baggage("confident.test_run.id", context=ctx)
+        print("Ending OTLP test run with run_id: ", run_id)
+        _tracer = check_tracer(tracer)
+        with _tracer.start_as_current_span(
+            "stop_otel_test_run", context=ctx
+        ) as span:
+            span.set_attribute("confident.test_run.id", run_id)

deepeval/dataset/test_run_tracer.py

@@ -0,0 +1,82 @@
+import os
+from typing import Optional
+from opentelemetry import baggage
+from opentelemetry.trace import Tracer as OTelTracer
+from opentelemetry.sdk.trace import SpanProcessor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+try:
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+
+    is_opentelemetry_installed = True
+except Exception:
+    is_opentelemetry_installed = False
+
+
+def is_opentelemetry_available():
+    if not is_opentelemetry_installed:
+        raise ImportError(
+            "OpenTelemetry SDK is not available. Please install it with `pip install opentelemetry-exporter-otlp-proto-http`."
+        )
+    return True
+
+
+from deepeval.confident.api import get_confident_api_key
+
+OTLP_ENDPOINT = (
+    os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+    else "https://otel.confident-ai.com"
+)
+# OTLP_ENDPOINT = "http://127.0.0.1:4318"
+
+# Module-level globals to be imported and used by other code
+GLOBAL_TEST_RUN_TRACER_PROVIDER: Optional[TracerProvider] = None
+GLOBAL_TEST_RUN_TRACER: Optional[OTelTracer] = None
+
+
+class RunIdSpanProcessor(SpanProcessor):
+    def on_start(self, span, parent_context):
+        run_id = baggage.get_baggage(
+            "confident.test_run.id", context=parent_context
+        )
+        if run_id:
+            span.set_attribute("confident.test_run.id", run_id)
+
+    def on_end(self, span) -> None:  # type: ignore[override]
+        # No-op
+        return None
+
+    def shutdown(self) -> None:  # type: ignore[override]
+        # No-op
+        return None
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:  # type: ignore[override]
+        # No-op
+        return True
+
+
+def init_global_test_run_tracer(api_key: Optional[str] = None):
+    is_opentelemetry_available()
+    api_key = get_confident_api_key()
+    if api_key is None:
+        raise ValueError("CONFIDENT_API_KEY is not set")
+
+    provider = TracerProvider()
+    exporter = OTLPSpanExporter(
+        endpoint=f"{OTLP_ENDPOINT}/v1/traces",
+        headers={"x-confident-api-key": api_key},
+    )
+    provider.add_span_processor(RunIdSpanProcessor())
+    provider.add_span_processor(BatchSpanProcessor(span_exporter=exporter))
+    tracer = provider.get_tracer("deepeval_tracer")
+
+    global GLOBAL_TEST_RUN_TRACER_PROVIDER
+    global GLOBAL_TEST_RUN_TRACER
+    GLOBAL_TEST_RUN_TRACER_PROVIDER = provider
+    GLOBAL_TEST_RUN_TRACER = tracer
+
+    return provider, tracer

deepeval/dataset/utils.py

@@ -2,6 +2,10 @@ from typing import List, Optional, Any
 import json
 import re
 
+from opentelemetry.trace import Tracer
+from opentelemetry import trace
+from opentelemetry.trace import NoOpTracerProvider
+
 from deepeval.dataset.api import Golden
 from deepeval.dataset.golden import ConversationalGolden
 from deepeval.test_case import LLMTestCase, ConversationalTestCase, Turn
@@ -151,3 +155,22 @@ def parse_turns(turns_str: str) -> List[Turn]:
             )
         )
     return res
+
+
+def check_tracer(tracer: Optional[Tracer] = None) -> Tracer:
+    if tracer:
+        return tracer
+    # Prefer module-level test-run tracer if available
+    try:
+        from deepeval.dataset.test_run_tracer import (
+            GLOBAL_TEST_RUN_TRACER,
+        )
+
+        if GLOBAL_TEST_RUN_TRACER is not None:
+            return GLOBAL_TEST_RUN_TRACER
+    except Exception:
+        raise RuntimeError(
+            "No global OpenTelemetry tracer provider is configured."  # TODO: link to docs
+        )
+
+    return GLOBAL_TEST_RUN_TRACER

deepeval/key_handler.py

@@ -1,12 +1,42 @@
 """File for handling API key"""
 
+import os
 import json
+import logging
+
 from enum import Enum
 from typing import Union
 
 from .constants import KEY_FILE, HIDDEN_DIR
 
 
+logger = logging.getLogger(__name__)
+
+
+SECRET_KEYS = {
+    # General providers
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    # Azure OpenAI
+    "AZURE_OPENAI_API_KEY",
+    # Google / Gemini
+    "GOOGLE_API_KEY",
+    # xAI Grok
+    "GROK_API_KEY",
+    # Moonshot
+    "MOONSHOT_API_KEY",
+    # DeepSeek
+    "DEEPSEEK_API_KEY",
+    # LiteLLM
+    "LITELLM_API_KEY",
+    # Local gateways (if any require keys)
+    "LOCAL_MODEL_API_KEY",
+    "LOCAL_EMBEDDING_API_KEY",
+}
+
+_WARNED_SECRET_KEYS = set()
+
+
 class KeyValues(Enum):
     # Confident AI
     API_KEY = "api_key"
@@ -50,6 +80,7 @@ class ModelKeyValues(Enum):
     OPENAI_MODEL_NAME = "OPENAI_MODEL_NAME"
     OPENAI_COST_PER_INPUT_TOKEN = "OPENAI_COST_PER_INPUT_TOKEN"
     OPENAI_COST_PER_OUTPUT_TOKEN = "OPENAI_COST_PER_OUTPUT_TOKEN"
+    OPENAI_API_KEY = "OPENAI_API_KEY"
     # Moonshot
     USE_MOONSHOT_MODEL = "USE_MOONSHOT_MODEL"
     MOONSHOT_MODEL_NAME = "MOONSHOT_MODEL_NAME"
@@ -79,10 +110,21 @@ class KeyFileHandler:
     def __init__(self):
         self.data = {}
 
+    def _ensure_dir(self):
+        os.makedirs(HIDDEN_DIR, exist_ok=True)
+
     def write_key(
         self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues], value
     ):
         """Appends or updates data in the hidden file"""
+
+        # hard stop on secrets: never write to disk
+        if key.value in SECRET_KEYS:
+            logger.warning(
+                f"{key} is blacklisted, refusing to persist. Keep your secrets in .env or .env.local instead"
+            )
+            return
+
         try:
             with open(f"{HIDDEN_DIR}/{KEY_FILE}", "r") as f:
                 # Load existing data
@@ -99,13 +141,15 @@ class KeyFileHandler:
         self.data[key.value] = value
 
         # Write the updated data back to the file
+        self._ensure_dir()
         with open(f"{HIDDEN_DIR}/{KEY_FILE}", "w") as f:
             json.dump(self.data, f)
 
     def fetch_data(
         self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
     ):
-        """Fetches the data from the hidden file"""
+        """Fetches the data from the hidden file.
+        NOTE: secrets in this file are deprecated; prefer env/.env."""
         try:
             with open(f"{HIDDEN_DIR}/{KEY_FILE}", "r") as f:
                 try:
@@ -116,7 +160,24 @@ class KeyFileHandler:
         except FileNotFoundError:
             # Handle the case when the file doesn't exist
             self.data = {}
-        return self.data.get(key.value)
+
+        value = self.data.get(key.value)
+
+        # Deprecation: warn only if we're actually returning a secret
+        if (
+            value is not None
+            and key.value in SECRET_KEYS
+            and key.value not in _WARNED_SECRET_KEYS
+        ):
+            logger.warning(
+                f"Reading secret '{key.value}' from legacy {HIDDEN_DIR}/{KEY_FILE}. "
+                "Persisting API keys in plaintext is deprecated. "
+                "Move this to your environment (.env / .env.local). "
+                "This fallback will be removed in a future release."
+            )
+            _WARNED_SECRET_KEYS.add(key.value)
+
+        return value
 
     def remove_key(
         self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
@@ -130,6 +191,7 @@ class KeyFileHandler:
                     # Handle corrupted JSON file
                     self.data = {}
             self.data.pop(key.value, None)  # Remove the key if it exists
+            self._ensure_dir()
             with open(f"{HIDDEN_DIR}/{KEY_FILE}", "w") as f:
                 json.dump(self.data, f)
         except FileNotFoundError:

deepeval/metrics/__init__.py

@@ -5,7 +5,8 @@ from .base_metric import (
     BaseArenaMetric,
 )
 
-from .dag.dag import DAGMetric
+from .dag.dag import DAGMetric, DeepAcyclicGraph
+from .conversational_dag.conversational_dag import ConversationalDAGMetric
 from .bias.bias import BiasMetric
 from .toxicity.toxicity import ToxicityMetric
 from .pii_leakage.pii_leakage import PIILeakageMetric
@@ -67,6 +68,8 @@ __all__ = [
     "ArenaGEval",
     "ConversationalGEval",
     "DAGMetric",
+    "DeepAcyclicGraph",
+    "ConversationalDAGMetric"
     # RAG metrics
     "AnswerRelevancyMetric",
     "FaithfulnessMetric",

deepeval/metrics/answer_relevancy/template.py

@@ -37,7 +37,7 @@ JSON:
 Please generate a list of JSON with two keys: `verdict` and `reason`.
 The 'verdict' key should STRICTLY be either a 'yes', 'idk' or 'no'. Answer 'yes' if the statement is relevant to addressing the original input, 'no' if the statement is irrelevant, and 'idk' if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).
 The 'reason' is the reason for the verdict.
-Provide a 'reason' ONLY if the answer is 'no'. 
+Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
 The provided statements are statements made in the actual output.
 
 **
@@ -53,7 +53,8 @@ Example statements:
     "Security features include fingerprint authentication and an encrypted SSD.",
     "Every purchase comes with a one-year warranty.",
     "24/7 customer support is included.",
-    "Pineapples taste great on pizza."
+    "Pineapples taste great on pizza.",
+    "The laptop is a Dell XPS 13."
 ]
 
 Example JSON:
@@ -79,6 +80,10 @@ Example JSON:
         {{
             "verdict": "no",
             "reason": "The statement about pineapples on pizza is completely irrelevant to the input, which asks about laptop features."
+        }},
+        {{
+            "verdict": "idk",
+            "reason": "The statement about the laptop being a Dell XPS 13 is not directly relevant to the input, but could be used as a supporting point to address the input."
         }}
     ]  
 }}

deepeval/metrics/conversational_dag/__init__.py

@@ -0,0 +1,7 @@
+from .nodes import (
+    ConversationalBaseNode,
+    ConversationalVerdictNode,
+    ConversationalTaskNode,
+    ConversationalBinaryJudgementNode,
+    ConversationalNonBinaryJudgementNode,
+)

deepeval/metrics/conversational_dag/conversational_dag.py

@@ -0,0 +1,139 @@
+from typing import Optional, Union
+from deepeval.metrics import BaseConversationalMetric
+from deepeval.test_case import (
+    ConversationalTestCase,
+)
+from deepeval.utils import get_or_create_event_loop
+from deepeval.metrics.utils import (
+    check_conversational_test_case_params,
+    construct_verbose_logs,
+    initialize_model,
+)
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.g_eval.schema import *
+from deepeval.metrics import DeepAcyclicGraph
+from deepeval.metrics.dag.utils import (
+    is_valid_dag_from_roots,
+    extract_required_params,
+    copy_graph,
+)
+
+
+class ConversationalDAGMetric(BaseConversationalMetric):
+
+    def __init__(
+        self,
+        name: str,
+        dag: DeepAcyclicGraph,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        threshold: float = 0.5,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+        _include_dag_suffix: bool = True,
+    ):
+        if (
+            is_valid_dag_from_roots(
+                root_nodes=dag.root_nodes, multiturn=dag.multiturn
+            )
+            == False
+        ):
+            raise ValueError("Cycle detected in DAG graph.")
+
+        self._verbose_steps: List[str] = []
+        self.dag = copy_graph(dag)
+        self.name = name
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.threshold = 1 if strict_mode else threshold
+        self.include_reason = include_reason
+        self.strict_mode = strict_mode
+        self.async_mode = async_mode
+        self.verbose_mode = verbose_mode
+        self._include_dag_suffix = _include_dag_suffix
+
+    def measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        check_conversational_test_case_params(
+            test_case,
+            extract_required_params(self.dag.root_nodes, multiturn=True),
+            self,
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                    )
+                )
+            else:
+                self.dag._execute(metric=self, test_case=test_case)
+                self.success = self.is_successful()
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        *self._verbose_steps,
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        check_conversational_test_case_params(
+            test_case,
+            extract_required_params(self.dag.root_nodes, multiturn=True),
+            self,
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            await self.dag._a_execute(metric=self, test_case=test_case)
+            self.success = self.is_successful()
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    *self._verbose_steps,
+                    f"Score: {self.score}\nReason: {self.reason}",
+                ],
+            )
+            return self.score
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        if self._include_dag_suffix:
+            return f"{self.name} [ConversationalDAG]"
+        else:
+            return self.name