deepeval 3.8.0__tar.gz → 3.8.2__tar.gz

{deepeval-3.8.0 → deepeval-3.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.8.0
+Version: 3.8.2
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.8.2/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.8.2"

{deepeval-3.8.0 → deepeval-3.8.2}/deepeval/annotation/annotation.py

@@ -14,7 +14,7 @@ def send_annotation(
     explanation: Optional[str] = None,
     user_id: Optional[str] = None,
     type: Optional[AnnotationType] = AnnotationType.THUMBS_RATING,
-) -> str:
+) -> None:
     api_annotation = APIAnnotation(
         rating=rating,
         traceUuid=trace_uuid,
@@ -50,7 +50,7 @@ async def a_send_annotation(
     explanation: Optional[str] = None,
     type: Optional[AnnotationType] = AnnotationType.THUMBS_RATING,
     user_id: Optional[str] = None,
-) -> str:
+) -> None:
     api_annotation = APIAnnotation(
         rating=rating,
         traceUuid=trace_uuid,

{deepeval-3.8.0 → deepeval-3.8.2}/deepeval/confident/api.py

@@ -26,16 +26,44 @@ API_BASE_URL_EU = "https://eu.api.confident-ai.com"
 retryable_exceptions = requests.exceptions.SSLError
 
 
+def _infer_region_from_api_key(api_key: Optional[str]) -> Optional[str]:
+    """
+    Infer region from Confident API key prefix.
+
+    Supported:
+      - confident_eu_... => "EU"
+      - confident_us_... => "US"
+
+    Returns None if prefix is not recognized or api_key is falsy.
+    """
+    if not api_key:
+        return None
+    key = api_key.strip().lower()
+    if key.startswith("confident_eu_"):
+        return "EU"
+    if key.startswith("confident_us_"):
+        return "US"
+    return None
+
+
 def get_base_api_url():
     s = get_settings()
     if s.CONFIDENT_BASE_URL:
         base_url = s.CONFIDENT_BASE_URL.rstrip("/")
         return base_url
+    # If the user has explicitly set a region, respect it.
     region = KEY_FILE_HANDLER.fetch_data(KeyValues.CONFIDENT_REGION)
-    if region == "EU":
+    if region:
+        return API_BASE_URL_EU if region == "EU" else API_BASE_URL
+
+    # Otherwise, infer region from the API key prefix.
+    api_key = get_confident_api_key()
+    inferred = _infer_region_from_api_key(api_key)
+    if inferred == "EU":
         return API_BASE_URL_EU
-    else:
-        return API_BASE_URL
+
+    # Default to US (backwards compatible)
+    return API_BASE_URL
 
 
 def get_confident_api_key() -> Optional[str]:

{deepeval-3.8.0 → deepeval-3.8.2}/deepeval/config/settings.py

@@ -447,6 +447,9 @@ class Settings(BaseSettings):
     AZURE_OPENAI_API_KEY: Optional[SecretStr] = Field(
         None, description="Azure OpenAI API key."
     )
+    AZURE_OPENAI_AD_TOKEN: Optional[SecretStr] = Field(
+        None, description="Azure OpenAI Ad Token."
+    )
     AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = Field(
         None, description="Azure OpenAI endpoint URL."
     )

{deepeval-3.8.0 → deepeval-3.8.2}/deepeval/dataset/dataset.py

@@ -84,9 +84,11 @@ class EvaluationDataset:
     def __init__(
         self,
         goldens: Union[List[Golden], List[ConversationalGolden]] = [],
+        confident_api_key: Optional[str] = None,
     ):
         self._alias = None
         self._id = None
+        self.confident_api_key = confident_api_key
         if len(goldens) > 0:
             self._multi_turn = (
                 True if isinstance(goldens[0], ConversationalGolden) else False
@@ -722,7 +724,7 @@ class EvaluationDataset:
                 "Unable to push empty dataset to Confident AI, there must be at least one golden in dataset."
             )
 
-        api = Api()
+        api = Api(api_key=self.confident_api_key)
         api_dataset = APIDataset(
             goldens=self.goldens if not self._multi_turn else None,
             conversationalGoldens=(self.goldens if self._multi_turn else None),
@@ -755,7 +757,7 @@ class EvaluationDataset:
         auto_convert_goldens_to_test_cases: bool = False,
         public: bool = False,
     ):
-        api = Api()
+        api = Api(api_key=self.confident_api_key)
         with capture_pull_dataset():
             with Progress(
                 SpinnerColumn(style="rgb(106,0,255)"),
@@ -839,7 +841,7 @@ class EvaluationDataset:
             raise ValueError(
                 f"Can't queue empty list of goldens to dataset with alias: {alias} on Confident AI."
             )
-        api = Api()
+        api = Api(api_key=self.confident_api_key)
 
         multi_turn = isinstance(goldens[0], ConversationalGolden)
 
@@ -871,7 +873,7 @@ class EvaluationDataset:
         self,
         alias: str,
     ):
-        api = Api()
+        api = Api(api_key=self.confident_api_key)
         api.send_request(
             method=HttpMethods.DELETE,
             endpoint=Endpoints.DATASET_ALIAS_ENDPOINT,

{deepeval-3.8.0 → deepeval-3.8.2}/deepeval/integrations/langchain/callback.py

@@ -1,3 +1,7 @@
+import logging
+import os
+import threading
+
 from typing import Any, Optional, List, Dict
 from uuid import UUID
 from time import perf_counter
@@ -20,6 +24,19 @@ from deepeval.tracing.types import (
 )
 from deepeval.telemetry import capture_tracing_integration
 
+# Debug logging for LangChain callbacks (enable with DEEPEVAL_DEBUG_LANGCHAIN_CALLBACKS=1)
+_DEBUG_CALLBACKS = os.environ.get(
+    "DEEPEVAL_DEBUG_LANGCHAIN_CALLBACKS", ""
+).lower() in ("1", "true", "yes")
+
+_logger = logging.getLogger(__name__)
+
+
+def _debug_log(msg: str):
+    if _DEBUG_CALLBACKS:
+        _logger.debug(f"[LangChain Callback] {msg}")
+
+
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
     from langchain_core.outputs import LLMResult
@@ -29,6 +46,7 @@ try:
     # contains langchain imports
     from deepeval.integrations.langchain.utils import (
         parse_prompts_to_messages,
+        convert_chat_messages_to_input,
         extract_name,
         safe_extract_model_name,
         safe_extract_token_usage,
@@ -50,6 +68,12 @@ def is_langchain_installed():
 
 
 class CallbackHandler(BaseCallbackHandler):
+    # When users create multiple CallbackHandler instances for the same logical
+    # conversation (same thread_id), we want spans to land on the same trace.
+    # Otherwise, each handler lazily creates its own trace, and multi-turn flows
+    # become multiple single-turn traces.
+    _thread_id_to_trace_uuid: Dict[str, str] = {}
+    _thread_id_lock = threading.Lock()
 
     def __init__(
         self,
@@ -74,13 +98,20 @@ class CallbackHandler(BaseCallbackHandler):
             self._parent_span = None
 
             # Stash trace metadata to apply once we know which trace we are using.
-            self._trace_init_fields: Dict[str, Any] = {
+            # _trace_init_fields is cleared after first apply to prevent re-applying
+            # on every callback within the same trace. _original_init_fields is kept
+            # permanently so we can re-apply when a new trace is created (e.g., in
+            # multi-turn scenarios where the previous trace was ended).
+            self._original_init_fields: Dict[str, Any] = {
                 "name": name,
                 "tags": tags,
                 "metadata": metadata,
                 "thread_id": thread_id,
                 "user_id": user_id,
             }
+            self._trace_init_fields: Dict[str, Any] = dict(
+                self._original_init_fields
+            )
 
             # Map LangChain run_id -> our span uuid for parent span restoration
             self._run_id_to_span_uuid: Dict[str, str] = {}
@@ -96,6 +127,34 @@ class CallbackHandler(BaseCallbackHandler):
         This is done lazily during actual callback execution to avoid context
         corruption when the handler is constructed outside the async task/context.
         """
+        # If the user provided a thread_id, attempt to reuse an existing trace for it.
+        # This makes multi-turn tests that use multiple CallbackHandler instances behave
+        # as expected: one trace containing multiple turns/spans.
+        thread_id = None
+        fields = self._trace_init_fields or {}
+        if fields.get("thread_id"):
+            thread_id = fields["thread_id"]
+        # In case _trace_init_fields has already been cleared, fall back to trace metadata.
+        if thread_id is None and self._trace is not None:
+            thread_id = self._trace.thread_id
+
+        if thread_id:
+            with self._thread_id_lock:
+                existing_uuid = self._thread_id_to_trace_uuid.get(thread_id)
+            if existing_uuid:
+                existing_trace = trace_manager.get_trace_by_uuid(existing_uuid)
+                if (
+                    existing_trace
+                    and existing_trace.uuid in trace_manager.active_traces
+                ):
+                    current_trace_context.set(existing_trace)
+                    self._trace = existing_trace
+                    self.trace_uuid = existing_trace.uuid
+                    # Lazily capture the observe parent span if present.
+                    if self._parent_span is None:
+                        self._parent_span = current_span_context.get()
+                    return existing_trace
+
         # Prefer current context trace if it is active.
         ctx_trace = current_trace_context.get()
         if ctx_trace and ctx_trace.uuid in trace_manager.active_traces:
@@ -107,6 +166,10 @@ class CallbackHandler(BaseCallbackHandler):
                 current_trace_context.set(trace)
             else:
                 # Otherwise, create a fresh trace now (in the right context).
+                # Restore _trace_init_fields from the original init fields so that
+                # the new trace gets the same name/tags/metadata as intended.
+                if not self._trace_init_fields and self._original_init_fields:
+                    self._trace_init_fields = dict(self._original_init_fields)
                 trace = trace_manager.start_new_trace()
                 current_trace_context.set(trace)
                 self._trace = trace
@@ -114,8 +177,18 @@ class CallbackHandler(BaseCallbackHandler):
         # Keep a copy for quick access.
         self.trace_uuid = trace.uuid
 
+        # Register this trace as the canonical trace for this thread_id (if provided).
+        # This allows other CallbackHandler instances created for the same thread_id
+        # to reuse the same trace instead of creating parallel traces.
+        fields = self._trace_init_fields or {}
+        tid = fields.get("thread_id") or trace.thread_id
+        if tid:
+            with self._thread_id_lock:
+                # Only set if absent to preserve the "first trace wins" behavior.
+                self._thread_id_to_trace_uuid.setdefault(tid, trace.uuid)
+
         # Apply stashed metadata once.
-        fields = getattr(self, "_trace_init_fields", None) or {}
+        fields = self._trace_init_fields or {}
         if fields:
             if fields.get("name") is not None:
                 trace.name = fields["name"]
@@ -202,6 +275,9 @@ class CallbackHandler(BaseCallbackHandler):
         metadata: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_chain_start: run_id={run_id}, parent_run_id={parent_run_id}, name={extract_name(serialized, **kwargs)}"
+        )
         # Create spans for all chains to establish proper parent-child hierarchy
         # This is important for LangGraph where there are nested chains
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
@@ -232,6 +308,9 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_chain_end: run_id={run_id}, parent_run_id={parent_run_id}"
+        )
         uuid_str = str(run_id)
         base_span = trace_manager.get_span_by_uuid(uuid_str)
         if base_span:
@@ -246,6 +325,59 @@ class CallbackHandler(BaseCallbackHandler):
                         trace.output = output
                 exit_current_context(uuid_str=uuid_str)
 
+    def on_chat_model_start(
+        self,
+        serialized: dict[str, Any],
+        messages: list[list[Any]],  # list[list[BaseMessage]]
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Handle chat model start callback. In LangChain v1, chat models emit
+        on_chat_model_start instead of on_llm_start. The on_llm_end callback
+        is still used for both.
+        """
+        _debug_log(
+            f"on_chat_model_start: run_id={run_id}, parent_run_id={parent_run_id}, messages_len={len(messages)}"
+        )
+
+        # Guard against double-counting if both on_llm_start and on_chat_model_start fire
+        uuid_str = str(run_id)
+        existing_span = trace_manager.get_span_by_uuid(uuid_str)
+        if existing_span is not None:
+            _debug_log(
+                f"on_chat_model_start: span already exists for run_id={run_id}, skipping"
+            )
+            return
+
+        with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
+            # Convert messages to our internal format using the shared helper
+            input_messages = convert_chat_messages_to_input(messages, **kwargs)
+
+            # Safe extraction of model name (handle None metadata)
+            md = metadata or {}
+            model = safe_extract_model_name(md, **kwargs)
+
+            llm_span: LlmSpan = enter_current_context(
+                uuid_str=uuid_str,
+                span_type="llm",
+                func_name=extract_name(serialized, **kwargs),
+            )
+            # Register this run_id -> span mapping for child callbacks
+            self._run_id_to_span_uuid[str(run_id)] = uuid_str
+
+            llm_span.input = input_messages
+            llm_span.model = model
+
+            # Extract metrics and prompt from metadata if provided, but don't mutate original
+            llm_span.metrics = md.get("metrics")
+            llm_span.metric_collection = md.get("metric_collection")
+            llm_span.prompt = md.get("prompt")
+
     def on_llm_start(
         self,
         serialized: dict[str, Any],
@@ -257,10 +389,25 @@ class CallbackHandler(BaseCallbackHandler):
         metadata: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_llm_start: run_id={run_id}, parent_run_id={parent_run_id}, prompts_len={len(prompts)}"
+        )
+
+        # Guard against double-counting if both on_llm_start and on_chat_model_start fire
+        uuid_str = str(run_id)
+        existing_span = trace_manager.get_span_by_uuid(uuid_str)
+        if existing_span is not None:
+            _debug_log(
+                f"on_llm_start: span already exists for run_id={run_id}, skipping"
+            )
+            return
+
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
-            uuid_str = str(run_id)
             input_messages = parse_prompts_to_messages(prompts, **kwargs)
-            model = safe_extract_model_name(metadata, **kwargs)
+
+            # Safe extraction of model name (handle None metadata)
+            md = metadata or {}
+            model = safe_extract_model_name(md, **kwargs)
 
             llm_span: LlmSpan = enter_current_context(
                 uuid_str=uuid_str,
@@ -272,12 +419,11 @@ class CallbackHandler(BaseCallbackHandler):
 
             llm_span.input = input_messages
             llm_span.model = model
-            metrics = metadata.pop("metrics", None)
-            metric_collection = metadata.pop("metric_collection", None)
-            prompt = metadata.pop("prompt", None)
-            llm_span.metrics = metrics
-            llm_span.metric_collection = metric_collection
-            llm_span.prompt = prompt
+
+            # Extract metrics and prompt from metadata if provided, but don't mutate original
+            llm_span.metrics = md.get("metrics")
+            llm_span.metric_collection = md.get("metric_collection")
+            llm_span.prompt = md.get("prompt")
 
     def on_llm_end(
         self,
@@ -287,9 +433,20 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
+        _debug_log(
+            f"on_llm_end: run_id={run_id}, parent_run_id={parent_run_id}, response_type={type(response).__name__}"
+        )
         uuid_str = str(run_id)
         llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
         if llm_span is None:
+            _debug_log(f"on_llm_end: NO SPAN FOUND for run_id={run_id}")
+            return
+
+        # Guard against double-finalization (if both on_llm_end and on_chat_model_end fire)
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_llm_end: span already finalized for run_id={run_id}, skipping"
+            )
             return
 
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
@@ -336,7 +493,6 @@ class CallbackHandler(BaseCallbackHandler):
                             )
 
             llm_span.model = model if model else llm_span.model
-            llm_span.input = llm_span.input
             llm_span.output = output
             llm_span.input_token_count = (
                 total_input_tokens if total_input_tokens > 0 else None
@@ -347,6 +503,121 @@ class CallbackHandler(BaseCallbackHandler):
 
             exit_current_context(uuid_str=uuid_str)
 
+    def on_chat_model_end(
+        self,
+        response: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Handle chat model end callback. This may be called instead of or
+        in addition to on_llm_end depending on the LangChain version.
+        """
+        _debug_log(
+            f"on_chat_model_end: run_id={run_id}, parent_run_id={parent_run_id}, response_type={type(response).__name__}"
+        )
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+        if llm_span is None:
+            _debug_log(f"on_chat_model_end: NO SPAN FOUND for run_id={run_id}")
+            return
+
+        # Guard against double-finalization, which could happen if both on_llm_end and on_chat_model_end fire
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_chat_model_end: span already finalized for run_id={run_id}, skipping"
+            )
+            return
+
+        with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
+            output = ""
+            total_input_tokens = 0
+            total_output_tokens = 0
+            model = None
+
+            # Handle LLMResult (same as on_llm_end)
+            if isinstance(response, LLMResult):
+                for generation in response.generations:
+                    for gen in generation:
+                        if isinstance(gen, ChatGeneration):
+                            if gen.message.response_metadata and isinstance(
+                                gen.message.response_metadata, dict
+                            ):
+                                model = gen.message.response_metadata.get(
+                                    "model_name"
+                                )
+                                input_tokens, output_tokens = (
+                                    safe_extract_token_usage(
+                                        gen.message.response_metadata
+                                    )
+                                )
+                                total_input_tokens += input_tokens
+                                total_output_tokens += output_tokens
+
+                            if isinstance(gen.message, AIMessage):
+                                ai_message = gen.message
+                                tool_calls = []
+                                for tool_call in ai_message.tool_calls:
+                                    tool_calls.append(
+                                        LlmToolCall(
+                                            name=tool_call["name"],
+                                            args=tool_call["args"],
+                                            id=tool_call["id"],
+                                        )
+                                    )
+                                output = LlmOutput(
+                                    role="AI",
+                                    content=ai_message.content,
+                                    tool_calls=tool_calls,
+                                )
+
+            llm_span.model = model if model else llm_span.model
+            llm_span.output = output
+            llm_span.input_token_count = (
+                total_input_tokens if total_input_tokens > 0 else None
+            )
+            llm_span.output_token_count = (
+                total_output_tokens if total_output_tokens > 0 else None
+            )
+
+            exit_current_context(uuid_str=uuid_str)
+
+    def on_chat_model_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """
+        Handle chat model error callback.
+        """
+        _debug_log(
+            f"on_chat_model_error: run_id={run_id}, parent_run_id={parent_run_id}, error={error}"
+        )
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+        if llm_span is None:
+            _debug_log(
+                f"on_chat_model_error: NO SPAN FOUND for run_id={run_id}"
+            )
+            return
+
+        # Guard against double-finalization
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_chat_model_error: span already finalized for run_id={run_id}, skipping"
+            )
+            return
+
+        with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
+            llm_span.status = TraceSpanStatus.ERRORED
+            llm_span.error = str(error)
+            exit_current_context(uuid_str=uuid_str)
+
     def on_llm_error(
         self,
         error: BaseException,
@@ -355,10 +626,22 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_llm_error: run_id={run_id}, parent_run_id={parent_run_id}, error={error}"
+        )
         uuid_str = str(run_id)
         llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
         if llm_span is None:
+            _debug_log(f"on_llm_error: NO SPAN FOUND for run_id={run_id}")
             return
+
+        # Guard against double-finalization
+        if llm_span.end_time is not None:
+            _debug_log(
+                f"on_llm_error: span already finalized for run_id={run_id}, skipping"
+            )
+            return
+
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             llm_span.status = TraceSpanStatus.ERRORED
             llm_span.error = str(error)
@@ -396,6 +679,9 @@ class CallbackHandler(BaseCallbackHandler):
         inputs: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
+        _debug_log(
+            f"on_tool_start: run_id={run_id}, parent_run_id={parent_run_id}, name={extract_name(serialized, **kwargs)}"
+        )
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             uuid_str = str(run_id)
 
@@ -418,6 +704,9 @@ class CallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,  # un-logged kwargs
     ) -> Any:
+        _debug_log(
+            f"on_tool_end: run_id={run_id}, parent_run_id={parent_run_id}"
+        )
         uuid_str = str(run_id)
         tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
         if tool_span is None:
@@ -485,20 +774,23 @@ class CallbackHandler(BaseCallbackHandler):
     ) -> Any:
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             uuid_str = str(run_id)
+            # Safe access to metadata (handle None)
+            md = metadata or {}
             retriever_span = enter_current_context(
                 uuid_str=uuid_str,
                 span_type="retriever",
                 func_name=extract_name(serialized, **kwargs),
                 observe_kwargs={
-                    "embedder": metadata.get(
-                        "ls_embedding_provider", "unknown"
-                    ),
+                    "embedder": md.get("ls_embedding_provider", "unknown"),
                 },
             )
             # Register this run_id -> span mapping for child callbacks
             self._run_id_to_span_uuid[str(run_id)] = uuid_str
             retriever_span.input = query
 
+            # Extract metric_collection from metadata if provided
+            retriever_span.metric_collection = md.get("metric_collection")
+
     def on_retriever_end(
         self,
         output: Any,
@@ -539,4 +831,4 @@ class CallbackHandler(BaseCallbackHandler):
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             retriever_span.status = TraceSpanStatus.ERRORED
             retriever_span.error = str(error)
-            exit_current_context(uuid_str=uuid_str)
+            exit_current_context(uuid_str=uuid_str)