deepeval 3.6.9__py3-none-any.whl → 3.7.0__py3-none-any.whl

deepeval/config/settings.py

@@ -9,10 +9,13 @@ Central config for DeepEval.
   type coercion.
 """
 
+import hashlib
+import json
 import logging
 import math
 import os
 import re
+import threading
 
 from dotenv import dotenv_values
 from pathlib import Path
@@ -22,6 +25,7 @@ from pydantic import (
     confloat,
     conint,
     field_validator,
+    model_validator,
     SecretStr,
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -39,6 +43,13 @@ from deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify
 logger = logging.getLogger(__name__)
 _SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
 
+# settings that were converted to computed fields with override counterparts
+_DEPRECATED_TO_OVERRIDE = {
+    "DEEPEVAL_PER_TASK_TIMEOUT_SECONDS": "DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE",
+    "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS": "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE",
+    "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS": "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE",
+}
+
 
 def _find_legacy_enum(env_key: str):
     from deepeval.key_handler import (
@@ -690,12 +701,119 @@ class Settings(BaseSettings):
             "CRITICAL, NOTSET, or a numeric logging level."
         )
 
+    @field_validator("DEEPEVAL_TELEMETRY_OPT_OUT", mode="before")
+    @classmethod
+    def _apply_telemetry_enabled_alias(cls, v):
+        """
+        Precedence (most secure):
+        - Any OFF signal wins if both are set:
+          - DEEPEVAL_TELEMETRY_OPT_OUT = truthy  -> OFF
+          - DEEPEVAL_TELEMETRY_ENABLED = falsy   -> OFF
+        - Else, ON signal:
+          - DEEPEVAL_TELEMETRY_OPT_OUT = falsy   -> ON
+          - DEEPEVAL_TELEMETRY_ENABLED = truthy  -> ON
+        - Else None (unset) -> ON
+        """
+
+        def normalize(x):
+            if x is None:
+                return None
+            s = str(x).strip()
+            return None if s == "" else parse_bool(s, default=False)
+
+        new_opt_out = normalize(v)  # True means OFF, False means ON
+        legacy_enabled = normalize(
+            os.getenv("DEEPEVAL_TELEMETRY_ENABLED")
+        )  # True means ON, False means OFF
+
+        off_signal = (new_opt_out is True) or (legacy_enabled is False)
+        on_signal = (new_opt_out is False) or (legacy_enabled is True)
+
+        # Conflict: simultaneous OFF and ON signals
+        if off_signal and on_signal:
+            # Only warn if verbose or debug
+            if parse_bool(
+                os.getenv("DEEPEVAL_VERBOSE_MODE"), default=False
+            ) or logger.isEnabledFor(logging.DEBUG):
+                logger.warning(
+                    "Conflicting telemetry flags detected: DEEPEVAL_TELEMETRY_OPT_OUT=%r, "
+                    "DEEPEVAL_TELEMETRY_ENABLED=%r. Defaulting to OFF.",
+                    new_opt_out,
+                    legacy_enabled,
+                )
+            return True  # OFF wins
+
+        # Clear winner
+        if off_signal:
+            return True  # OFF
+        if on_signal:
+            return False  # ON
+
+        # Unset means ON
+        return False
+
+    @model_validator(mode="after")
+    def _apply_deprecated_computed_env_aliases(self):
+        """
+        Backwards compatibility courtesy:
+        - If users still set a deprecated computed field in the environment,
+          emit a deprecation warning and mirror its value into the matching
+          *_OVERRIDE field (unless the override is already set).
+        - Override always wins if both are present.
+        """
+        for old_key, override_key in _DEPRECATED_TO_OVERRIDE.items():
+            raw = os.getenv(old_key)
+            if raw is None or str(raw).strip() == "":
+                continue
+
+            # if override already set, ignore the deprecated one but log a warning
+            if getattr(self, override_key) is not None:
+                logger.warning(
+                    "Config deprecation: %s is deprecated and was ignored because %s "
+                    "is already set. Please remove %s and use %s going forward.",
+                    old_key,
+                    override_key,
+                    old_key,
+                    override_key,
+                )
+                continue
+
+            # apply the deprecated value into the override field.
+            try:
+                # let pydantic coerce the string to the target type on assignment
+                setattr(self, override_key, raw)
+                logger.warning(
+                    "Config deprecation: %s is deprecated. Its value (%r) was applied to %s. "
+                    "Please migrate to %s and remove %s from your environment.",
+                    old_key,
+                    raw,
+                    override_key,
+                    override_key,
+                    old_key,
+                )
+            except Exception as e:
+                # do not let exception bubble up, just warn
+                logger.warning(
+                    "Config deprecation: %s is deprecated and could not be applied to %s "
+                    "(value=%r): %s",
+                    old_key,
+                    override_key,
+                    raw,
+                    e,
+                )
+        return self
+
     #######################
     # Persistence support #
     #######################
     class _SettingsEditCtx:
+        # TODO: will generate this list in future PR
         COMPUTED_FIELDS: frozenset[str] = frozenset(
-            {"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS"}
+            {
+                "DEEPEVAL_PER_TASK_TIMEOUT_SECONDS",
+                "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS",
+                "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS",
+            }
         )
 
         def __init__(
@@ -880,16 +998,39 @@ class Settings(BaseSettings):
 
 
 _settings_singleton: Optional[Settings] = None
+_settings_env_fingerprint: "str | None" = None
+_settings_lock = threading.RLock()
+
+
+def _calc_env_fingerprint() -> str:
+    env = os.environ.copy()
+    # must hash in a stable order.
+    keys = sorted(
+        key
+        for key in Settings.model_fields.keys()
+        if key != "_DEPRECATED_TELEMETRY_ENABLED"  # exclude deprecated
+    )
+    # encode as triples: (key, present?, value)
+    items = [(k, k in env, env.get(k)) for k in keys]
+    payload = json.dumps(items, ensure_ascii=False, separators=(",", ":"))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
 
 def get_settings() -> Settings:
-    global _settings_singleton
-    if _settings_singleton is None:
-        _settings_singleton = Settings()
-        from deepeval.config.logging import apply_deepeval_log_level
+    global _settings_singleton, _settings_env_fingerprint
+    fingerprint = _calc_env_fingerprint()
+
+    with _settings_lock:
+        if (
+            _settings_singleton is None
+            or _settings_env_fingerprint != fingerprint
+        ):
+            _settings_singleton = Settings()
+            _settings_env_fingerprint = fingerprint
+            from deepeval.config.logging import apply_deepeval_log_level
 
-        apply_deepeval_log_level()
-    return _settings_singleton
+            apply_deepeval_log_level()
+        return _settings_singleton
 
 
 def reset_settings(*, reload_dotenv: bool = False) -> Settings:
@@ -905,8 +1046,10 @@ def reset_settings(*, reload_dotenv: bool = False) -> Settings:
     Returns:
         The fresh Settings instance.
     """
-    global _settings_singleton
-    if reload_dotenv:
-        autoload_dotenv()
-    _settings_singleton = None
+    global _settings_singleton, _settings_env_fingerprint
+    with _settings_lock:
+        if reload_dotenv:
+            autoload_dotenv()
+        _settings_singleton = None
+        _settings_env_fingerprint = None
     return get_settings()

deepeval/config/settings_manager.py

@@ -4,6 +4,7 @@ dotenv file. Also syncs os.environ, handles unsets, and warns on unknown fields.
 Primary entrypoint: update_settings_and_persist.
 """
 
+import json
 import logging
 import os
 
@@ -33,6 +34,9 @@ def _normalize_for_env(val: Any) -> Optional[str]:
         return val.get_secret_value()
     if isinstance(val, bool):
         return bool_to_env_str(val)
+    # encode sequences as JSON so Settings can parse them back reliably.
+    if isinstance(val, (list, tuple, set)):
+        return json.dumps(list(val))
     return str(val)
 
 

deepeval/integrations/crewai/handler.py

@@ -23,6 +23,8 @@ try:
         AgentExecutionCompletedEvent,
         ToolUsageStartedEvent,
         ToolUsageFinishedEvent,
+        KnowledgeRetrievalStartedEvent,
+        KnowledgeRetrievalCompletedEvent,
     )
 
     crewai_installed = True
@@ -69,6 +71,14 @@ class CrewAIEventsListener(BaseEventListener):
 
         return execution_id
 
+    @staticmethod
+    def get_knowledge_execution_id(source, event) -> str:
+        source_id = id(source)
+        agent_id = id(event.agent) if hasattr(event, "agent") else "unknown"
+        execution_id = f"_knowledge_{source_id}_{agent_id}"
+
+        return execution_id
+
     def setup_listeners(self, crewai_event_bus):
         @crewai_event_bus.on(CrewKickoffStartedEvent)
         def on_crew_started(source, event: CrewKickoffStartedEvent):
@@ -161,6 +171,32 @@ class CrewAIEventsListener(BaseEventListener):
                     current_span.output = event.output
                 observer.__exit__(None, None, None)
 
+        @crewai_event_bus.on(KnowledgeRetrievalStartedEvent)
+        def on_knowledge_started(source, event: KnowledgeRetrievalStartedEvent):
+            observer = Observer(
+                span_type="tool",
+                func_name="knowledge_retrieval",
+                function_kwargs={},
+            )
+            self.span_observers[
+                self.get_knowledge_execution_id(source, event)
+            ] = observer
+            observer.__enter__()
+
+        @crewai_event_bus.on(KnowledgeRetrievalCompletedEvent)
+        def on_knowledge_completed(
+            source, event: KnowledgeRetrievalCompletedEvent
+        ):
+            observer = self.span_observers.pop(
+                self.get_knowledge_execution_id(source, event)
+            )
+            if observer:
+                current_span = current_span_context.get()
+                if current_span:
+                    current_span.input = event.query
+                    current_span.output = event.retrieved_knowledge
+                observer.__exit__(None, None, None)
+
 
 def instrument_crewai(api_key: Optional[str] = None):
     is_crewai_installed()

deepeval/integrations/langchain/callback.py

@@ -1,12 +1,15 @@
 from typing import Any, Optional, List, Dict
 from uuid import UUID
 from time import perf_counter
-from deepeval.tracing.context import current_trace_context
+
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.test_case.llm_test_case import ToolCall
 from deepeval.tracing.types import (
     LlmOutput,
     LlmToolCall,
 )
 from deepeval.metrics import BaseMetric
+from deepeval.tracing.utils import prepare_tool_call_input_parameters
 
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
@@ -266,12 +269,34 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
-
         uuid_str = str(run_id)
         tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
         tool_span.output = output
         exit_current_context(uuid_str=uuid_str)
 
+        # set the tools called in the parent span as well as on the trace level
+        tool_call = ToolCall(
+            name=tool_span.name,
+            description=tool_span.description,
+            output=output,
+            input_parameters=prepare_tool_call_input_parameters(
+                tool_span.input
+            ),
+        )
+        parent_span = current_span_context.get()
+        if parent_span:
+            if parent_span.tools_called is None:
+                parent_span.tools_called = []
+
+            parent_span.tools_called.append(tool_call)
+
+        trace = current_trace_context.get()
+        if trace:
+            if trace.tools_called is None:
+                trace.tools_called = []
+
+            trace.tools_called.append(tool_call)
+
     def on_tool_error(
         self,
         error: BaseException,

deepeval/integrations/llama_index/handler.py

@@ -2,13 +2,28 @@ from typing import Any, Dict, Optional
 import inspect
 from time import perf_counter
 import uuid
+
+from llama_index.core.agent.workflow.workflow_events import (
+    AgentWorkflowStartEvent,
+)
+from deepeval.integrations.llama_index.utils import (
+    extract_output_from_llm_chat_end_event,
+)
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing import trace_manager
-from deepeval.tracing.types import AgentSpan, BaseSpan, LlmSpan, TraceSpanStatus
+from deepeval.tracing.types import (
+    ToolSpan,
+    AgentSpan,
+    BaseSpan,
+    LlmSpan,
+    TraceSpanStatus,
+)
 from deepeval.tracing.trace_context import (
     current_llm_context,
     current_agent_context,
 )
+from deepeval.test_case import ToolCall
+from deepeval.tracing.utils import make_json_serializable
 
 try:
     from llama_index.core.instrumentation.events.base import BaseEvent
@@ -89,6 +104,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     if llm_span_context
                     else None
                 ),
+                prompt=llm_span_context.prompt if llm_span_context else None,
             )
             trace_manager.add_span(llm_span)
             trace_manager.add_span_to_trace(llm_span)
@@ -106,7 +122,9 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     llm_span.status = TraceSpanStatus.SUCCESS
                     llm_span.end_time = perf_counter()
                     llm_span.input = llm_span.input
-                    llm_span.output = event.response.message.blocks[0].text
+                    llm_span.output = extract_output_from_llm_chat_end_event(
+                        event
+                    )
                     trace_manager.remove_span(llm_span.uuid)
                     del self.open_ai_astream_to_llm_span_map[event.span_id]
 
@@ -151,6 +169,14 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
         # conditions to qualify as agent start run span
         if method_name == "run":
             agent_span_context = current_agent_context.get()
+            start_event = bound_args.arguments.get("start_event")
+
+            if start_event and isinstance(start_event, AgentWorkflowStartEvent):
+                input = start_event.model_dump()
+
+            else:
+                input = bound_args.arguments
+
             span = AgentSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -159,7 +185,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 parent_uuid=parent_span_id,
                 start_time=perf_counter(),
                 name="Agent",  # TODO: decide the name of the span
-                input=bound_args.arguments,
+                input=input,
                 metrics=(
                     agent_span_context.metrics if agent_span_context else None
                 ),
@@ -169,7 +195,17 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     else None
                 ),
             )
-
+        elif method_name == "acall":
+            span = ToolSpan(
+                uuid=id_,
+                status=TraceSpanStatus.IN_PROGRESS,
+                children=[],
+                trace_uuid=trace_uuid,
+                parent_uuid=parent_span_id,
+                start_time=perf_counter(),
+                input=bound_args.arguments,
+                name="Tool",
+            )
         # prepare input test case params for the span
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments
@@ -192,10 +228,28 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
         if base_span is None:
             return None
 
+        class_name, method_name = parse_id(id_)
+        if method_name == "call_tool":
+            output_json = make_json_serializable(result)
+            if output_json and isinstance(output_json, dict):
+                if base_span.tools_called is None:
+                    base_span.tools_called = []
+                base_span.tools_called.append(
+                    ToolCall(
+                        name=output_json.get("tool_name", "Tool"),
+                        input_parameters=output_json.get("tool_kwargs", {}),
+                        output=output_json.get("tool_output", {}),
+                    )
+                )
         base_span.end_time = perf_counter()
         base_span.status = TraceSpanStatus.SUCCESS
         base_span.output = result
 
+        if isinstance(base_span, ToolSpan):
+            result_json = make_json_serializable(result)
+            if result_json and isinstance(result_json, dict):
+                base_span.name = result_json.get("tool_name", "Tool")
+
         if base_span.llm_test_case:
             class_name, method_name = parse_id(id_)
             prepare_output_llm_test_case_params(

deepeval/integrations/llama_index/utils.py

@@ -1,3 +1,4 @@
+from llama_index.core.instrumentation.events.llm import LLMChatEndEvent
 from deepeval.test_case.llm_test_case import LLMTestCase, ToolCall
 from deepeval.tracing.types import BaseSpan
 from typing import Any
@@ -81,3 +82,26 @@ def prepare_output_llm_test_case_params(
                 )
 
             span.llm_test_case.tools_called = tool_calls
+
+
+def extract_output_from_llm_chat_end_event(event: LLMChatEndEvent) -> list:
+    messages = []
+    for msg in event.response.message.blocks:
+        if msg.block_type == "text":
+            messages.append(
+                {
+                    "role": event.response.message.role.value,
+                    "content": msg.text,
+                }
+            )
+        elif msg.block_type == "tool_call":
+            messages.append(
+                {
+                    "name": msg.tool_name,
+                    "input_parameters": msg.tool_kwargs,
+                    "id": msg.tool_call_id,
+                }
+            )
+        else:
+            messages.append(msg.model_dump())
+    return messages

deepeval/metrics/__init__.py

@@ -8,6 +8,8 @@ from .base_metric import (
 from .dag.dag import DAGMetric, DeepAcyclicGraph
 from .conversational_dag.conversational_dag import ConversationalDAGMetric
 from .bias.bias import BiasMetric
+from .exact_match.exact_match import ExactMatchMetric
+from .pattern_match.pattern_match import PatternMatchMetric
 from .toxicity.toxicity import ToxicityMetric
 from .pii_leakage.pii_leakage import PIILeakageMetric
 from .non_advice.non_advice import NonAdviceMetric
@@ -69,6 +71,9 @@ __all__ = [
     "BaseConversationalMetric",
     "BaseMultimodalMetric",
     "BaseArenaMetric",
+    # Non-LLM metrics
+    "ExactMatchMetric",
+    "PatternMatchMetric",
     # Core metrics
     "GEval",
     "ArenaGEval",

deepeval/metrics/exact_match/exact_match.py

@@ -0,0 +1,94 @@
+from typing import List
+
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    check_llm_test_case_params,
+    construct_verbose_logs,
+)
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+
+class ExactMatchMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.EXPECTED_OUTPUT,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 1,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = threshold
+        self.verbose_mode = verbose_mode
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            expected = test_case.expected_output.strip()
+            actual = test_case.actual_output.strip()
+
+            if expected == actual:
+                self.score = self.precision = self.recall = self.f1 = 1.0
+                self.reason = (
+                    "The actual and expected outputs are exact matches."
+                )
+            else:
+                self.score = self.precision = self.recall = self.f1 = 0.0
+                self.reason = "The actual and expected outputs are different."
+
+            self.success = self.score >= self.threshold
+
+            if self.verbose_mode:
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Score: {self.score:.2f}",
+                        f"Reason: {self.reason}",
+                    ],
+                )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        return self.measure(
+            test_case,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        )
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Exact Match"