deepeval 3.6.3__py3-none-any.whl → 3.6.5__py3-none-any.whl

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -1,7 +1,19 @@
 import json
+import logging
 import os
 from typing import Literal, Optional, List
 
+from deepeval.config.settings import get_settings
+from deepeval.confident.api import get_confident_api_key
+from deepeval.prompt import Prompt
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import Trace
+from deepeval.tracing.otel.utils import to_hex_string
+
+
+logger = logging.getLogger(__name__)
+
+
 try:
     from pydantic_ai.models.instrumented import InstrumentationSettings
     from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
@@ -11,7 +23,20 @@ try:
     )
 
     dependency_installed = True
-except:
+except ImportError as e:
+    if get_settings().DEEPEVAL_VERBOSE_MODE:
+        if isinstance(e, ModuleNotFoundError):
+            logger.warning(
+                "Optional tracing dependency not installed: %s",
+                e.name,
+                stacklevel=2,
+            )
+        else:
+            logger.warning(
+                "Optional tracing import failed: %s",
+                e,
+                stacklevel=2,
+            )
     dependency_installed = False
 
 
@@ -25,6 +50,10 @@ def is_dependency_installed():
 
 from deepeval.confident.api import get_confident_api_key
 from deepeval.prompt import Prompt
+from deepeval.tracing.otel.test_exporter import test_exporter
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import Trace
+from deepeval.tracing.otel.utils import to_hex_string
 
 # OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
 OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
@@ -37,6 +66,12 @@ class SpanInterceptor(SpanProcessor):
 
     def on_start(self, span, parent_context):
 
+        # set trace uuid
+        _current_trace_context = current_trace_context.get()
+        if _current_trace_context and isinstance(_current_trace_context, Trace):
+            _otel_trace_id = span.get_span_context().trace_id
+            _current_trace_context.uuid = to_hex_string(_otel_trace_id, 32)
+
         # set trace attributes
         if self.settings.thread_id:
             span.set_attribute(
@@ -148,8 +183,9 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         confident_prompt: Optional[Prompt] = None,
         llm_metric_collection: Optional[str] = None,
         agent_metric_collection: Optional[str] = None,
-        tool_metric_collection_map: dict = {},
+        tool_metric_collection_map: Optional[dict] = None,
         trace_metric_collection: Optional[str] = None,
+        is_test_mode: Optional[bool] = False,
     ):
         is_dependency_installed()
 
@@ -162,7 +198,7 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         ]:
             self.environment = _environment
 
-        self.tool_metric_collection_map = tool_metric_collection_map
+        self.tool_metric_collection_map = tool_metric_collection_map or {}
         self.name = name
         self.thread_id = thread_id
         self.user_id = user_id
@@ -185,12 +221,15 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         span_interceptor = SpanInterceptor(self)
         trace_provider.add_span_processor(span_interceptor)
 
-        trace_provider.add_span_processor(
-            BatchSpanProcessor(
-                OTLPSpanExporter(
-                    endpoint=OTLP_ENDPOINT,
-                    headers={"x-confident-api-key": api_key},
+        if is_test_mode:
+            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+        else:
+            trace_provider.add_span_processor(
+                BatchSpanProcessor(
+                    OTLPSpanExporter(
+                        endpoint=OTLP_ENDPOINT,
+                        headers={"x-confident-api-key": api_key},
+                    )
                 )
             )
-        )
         super().__init__(tracer_provider=trace_provider)

deepeval/metrics/faithfulness/faithfulness.py

@@ -41,6 +41,7 @@ class FaithfulnessMetric(BaseMetric):
         strict_mode: bool = False,
         verbose_mode: bool = False,
         truths_extraction_limit: Optional[int] = None,
+        penalize_ambiguous_claims: bool = False,
         evaluation_template: Type[FaithfulnessTemplate] = FaithfulnessTemplate,
     ):
         self.threshold = 1 if strict_mode else threshold
@@ -51,6 +52,7 @@ class FaithfulnessMetric(BaseMetric):
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
         self.evaluation_template = evaluation_template
+        self.penalize_ambiguous_claims = penalize_ambiguous_claims
 
         self.truths_extraction_limit = truths_extraction_limit
         if self.truths_extraction_limit is not None:
@@ -329,6 +331,12 @@ class FaithfulnessMetric(BaseMetric):
             if verdict.verdict.strip().lower() != "no":
                 faithfulness_count += 1
 
+            if (
+                self.penalize_ambiguous_claims
+                and verdict.verdict.strip().lower() == "idk"
+            ):
+                faithfulness_count -= 1
+
         score = faithfulness_count / number_of_verdicts
         return 0 if self.strict_mode and score < self.threshold else score
 

deepeval/prompt/prompt.py

@@ -8,6 +8,7 @@ import os
 from pydantic import BaseModel
 import asyncio
 import portalocker
+import threading
 
 from deepeval.prompt.api import (
     PromptHttpResponse,
@@ -20,15 +21,39 @@ from deepeval.prompt.api import (
 from deepeval.prompt.utils import interpolate_text
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.constants import HIDDEN_DIR
-from deepeval.utils import (
-    get_or_create_event_loop,
-    get_or_create_general_event_loop,
-)
 
 CACHE_FILE_NAME = f"{HIDDEN_DIR}/.deepeval-prompt-cache.json"
 VERSION_CACHE_KEY = "version"
 LABEL_CACHE_KEY = "label"
 
+# Global background event loop for polling
+_polling_loop: Optional[asyncio.AbstractEventLoop] = None
+_polling_thread: Optional[threading.Thread] = None
+_polling_loop_lock = threading.Lock()
+
+
+def _get_or_create_polling_loop() -> asyncio.AbstractEventLoop:
+    """Get or create a background event loop for polling that runs in a daemon thread."""
+    global _polling_loop, _polling_thread
+
+    with _polling_loop_lock:
+        if _polling_loop is None or not _polling_loop.is_running():
+
+            def run_loop():
+                global _polling_loop
+                _polling_loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(_polling_loop)
+                _polling_loop.run_forever()
+
+            _polling_thread = threading.Thread(target=run_loop, daemon=True)
+            _polling_thread.start()
+
+            # Wait for loop to be ready
+            while _polling_loop is None:
+                time.sleep(0.01)
+
+        return _polling_loop
+
 
 class CustomEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -80,11 +105,22 @@ class Prompt:
         self._version = None
         self._polling_tasks: Dict[str, Dict[str, asyncio.Task]] = {}
         self._refresh_map: Dict[str, Dict[str, int]] = {}
+        self._lock = (
+            threading.Lock()
+        )  # Protect instance attributes from race conditions
         if template:
             self._type = PromptType.TEXT
         elif messages_template:
             self._type = PromptType.LIST
 
+    def __del__(self):
+        """Cleanup polling tasks when instance is destroyed"""
+        try:
+            self._stop_polling()
+        except Exception:
+            # Suppress exceptions during cleanup to avoid issues in interpreter shutdown
+            pass
+
     @property
     def version(self):
         if self._version is not None and self._version != "latest":
@@ -100,33 +136,37 @@ class Prompt:
         self._version = value
 
     def interpolate(self, **kwargs):
-        if self._type == PromptType.TEXT:
-            if self._text_template is None:
+        with self._lock:
+            prompt_type = self._type
+            text_template = self._text_template
+            messages_template = self._messages_template
+            interpolation_type = self._interpolation_type
+
+        if prompt_type == PromptType.TEXT:
+            if text_template is None:
                 raise TypeError(
                     "Unable to interpolate empty prompt template. Please pull a prompt from Confident AI or set template manually to continue."
                 )
 
-            return interpolate_text(
-                self._interpolation_type, self._text_template, **kwargs
-            )
+            return interpolate_text(interpolation_type, text_template, **kwargs)
 
-        elif self._type == PromptType.LIST:
-            if self._messages_template is None:
+        elif prompt_type == PromptType.LIST:
+            if messages_template is None:
                 raise TypeError(
                     "Unable to interpolate empty prompt template messages. Please pull a prompt from Confident AI or set template manually to continue."
                 )
 
             interpolated_messages = []
-            for message in self._messages_template:
+            for message in messages_template:
                 interpolated_content = interpolate_text(
-                    self._interpolation_type, message.content, **kwargs
+                    interpolation_type, message.content, **kwargs
                 )
                 interpolated_messages.append(
                     {"role": message.role, "content": interpolated_content}
                 )
             return interpolated_messages
         else:
-            raise ValueError(f"Unsupported prompt type: {self._type}")
+            raise ValueError(f"Unsupported prompt type: {prompt_type}")
 
     def _get_versions(self) -> List:
         if self.alias is None:
@@ -272,15 +312,16 @@ class Prompt:
         if not cached_prompt:
             raise ValueError("Unable to fetch prompt and load from cache")
 
-        self.version = cached_prompt.version
-        self.label = cached_prompt.label
-        self._text_template = cached_prompt.template
-        self._messages_template = cached_prompt.messages_template
-        self._prompt_version_id = cached_prompt.prompt_version_id
-        self._type = PromptType(cached_prompt.type)
-        self._interpolation_type = PromptInterpolationType(
-            cached_prompt.interpolation_type
-        )
+        with self._lock:
+            self.version = cached_prompt.version
+            self.label = cached_prompt.label
+            self._text_template = cached_prompt.template
+            self._messages_template = cached_prompt.messages_template
+            self._prompt_version_id = cached_prompt.prompt_version_id
+            self._type = PromptType(cached_prompt.type)
+            self._interpolation_type = PromptInterpolationType(
+                cached_prompt.interpolation_type
+            )
 
         end_time = time.perf_counter()
         time_taken = format(end_time - start_time, ".2f")
@@ -300,7 +341,6 @@ class Prompt:
     ):
         should_write_on_first_fetch = False
         if refresh:
-            default_to_cache = True
             # Check if we need to bootstrap the cache
             cached_prompt = self._read_from_cache(
                 self.alias, version=version, label=label
@@ -316,12 +356,10 @@ class Prompt:
             )
 
         # Manage background prompt polling
-        loop = get_or_create_general_event_loop()
-        if loop.is_running():
-            loop.create_task(self.create_polling_task(version, label, refresh))
-        else:
-            loop.run_until_complete(
-                self.create_polling_task(version, label, refresh)
+        if refresh:
+            loop = _get_or_create_polling_loop()
+            asyncio.run_coroutine_threadsafe(
+                self.create_polling_task(version, label, refresh), loop
             )
 
         if default_to_cache:
@@ -330,15 +368,20 @@ class Prompt:
                     self.alias, version=version, label=label
                 )
                 if cached_prompt:
-                    self.version = cached_prompt.version
-                    self.label = cached_prompt.label
-                    self._text_template = cached_prompt.template
-                    self._messages_template = cached_prompt.messages_template
-                    self._prompt_version_id = cached_prompt.prompt_version_id
-                    self._type = PromptType(cached_prompt.type)
-                    self._interpolation_type = PromptInterpolationType(
-                        cached_prompt.interpolation_type
-                    )
+                    with self._lock:
+                        self.version = cached_prompt.version
+                        self.label = cached_prompt.label
+                        self._text_template = cached_prompt.template
+                        self._messages_template = (
+                            cached_prompt.messages_template
+                        )
+                        self._prompt_version_id = (
+                            cached_prompt.prompt_version_id
+                        )
+                        self._type = PromptType(cached_prompt.type)
+                        self._interpolation_type = PromptInterpolationType(
+                            cached_prompt.interpolation_type
+                        )
                     return
             except:
                 pass
@@ -402,13 +445,14 @@ class Prompt:
                     return
                 raise
 
-            self.version = response.version
-            self.label = response.label
-            self._text_template = response.text
-            self._messages_template = response.messages
-            self._prompt_version_id = response.id
-            self._type = response.type
-            self._interpolation_type = response.interpolation_type
+            with self._lock:
+                self.version = response.version
+                self.label = response.label
+                self._text_template = response.text
+                self._messages_template = response.messages
+                self._prompt_version_id = response.id
+                self._type = response.type
+                self._interpolation_type = response.interpolation_type
 
             end_time = time.perf_counter()
             time_taken = format(end_time - start_time, ".2f")
@@ -483,11 +527,7 @@ class Prompt:
         version: Optional[str],
         label: Optional[str],
         refresh: Optional[int] = 60,
-        default_to_cache: bool = True,
     ):
-        if version is None and label is None:
-            return
-
         # If polling task doesn't exist, start it
         CACHE_KEY = LABEL_CACHE_KEY if label else VERSION_CACHE_KEY
         cache_value = label if label else version
@@ -506,9 +546,7 @@ class Prompt:
             self._refresh_map[CACHE_KEY][cache_value] = refresh
             if not polling_task:
                 self._polling_tasks[CACHE_KEY][cache_value] = (
-                    asyncio.create_task(
-                        self.poll(version, label, default_to_cache)
-                    )
+                    asyncio.create_task(self.poll(version, label))
                 )
 
         # If invalid `refresh`, stop the task
@@ -524,24 +562,12 @@ class Prompt:
         self,
         version: Optional[str] = None,
         label: Optional[str] = None,
-        default_to_cache: bool = True,
     ):
+        CACHE_KEY = LABEL_CACHE_KEY if label else VERSION_CACHE_KEY
+        cache_value = label if label else version
+
         while True:
-            if default_to_cache:
-                cached_prompt = self._read_from_cache(
-                    self.alias, version=version, label=label
-                )
-                if cached_prompt:
-                    self.version = cached_prompt.version
-                    self.label = cached_prompt.label
-                    self._text_template = cached_prompt.template
-                    self._messages_template = cached_prompt.messages_template
-                    self._prompt_version_id = cached_prompt.prompt_version_id
-                    self._type = PromptType(cached_prompt.type)
-                    self._interpolation_type = PromptInterpolationType(
-                        cached_prompt.interpolation_type
-                    )
-                    return
+            await asyncio.sleep(self._refresh_map[CACHE_KEY][cache_value])
 
             api = Api()
             try:
@@ -573,22 +599,43 @@ class Prompt:
                     type=data["type"],
                     interpolation_type=data["interpolationType"],
                 )
-                if default_to_cache:
-                    self._write_to_cache(
-                        cache_key=(
-                            LABEL_CACHE_KEY if label else VERSION_CACHE_KEY
-                        ),
-                        version=response.version,
-                        label=response.label,
-                        text_template=response.text,
-                        messages_template=response.messages,
-                        prompt_version_id=response.id,
-                        type=response.type,
-                        interpolation_type=response.interpolation_type,
-                    )
-            except Exception as e:
+
+                # Update the cache with fresh data from server
+                self._write_to_cache(
+                    cache_key=CACHE_KEY,
+                    version=response.version,
+                    label=response.label,
+                    text_template=response.text,
+                    messages_template=response.messages,
+                    prompt_version_id=response.id,
+                    type=response.type,
+                    interpolation_type=response.interpolation_type,
+                )
+
+                # Update in-memory properties with fresh data (thread-safe)
+                with self._lock:
+                    self.version = response.version
+                    self.label = response.label
+                    self._text_template = response.text
+                    self._messages_template = response.messages
+                    self._prompt_version_id = response.id
+                    self._type = response.type
+                    self._interpolation_type = response.interpolation_type
+
+            except Exception:
                 pass
 
-            CACHE_KEY = LABEL_CACHE_KEY if label else VERSION_CACHE_KEY
-            cache_value = label if label else version
-            await asyncio.sleep(self._refresh_map[CACHE_KEY][cache_value])
+    def _stop_polling(self):
+        loop = _polling_loop
+        if not loop or not loop.is_running():
+            return
+
+        # Stop all polling tasks
+        for ck in list(self._polling_tasks.keys()):
+            for cv in list(self._polling_tasks[ck].keys()):
+                task = self._polling_tasks[ck][cv]
+                if task and not task.done():
+                    loop.call_soon_threadsafe(task.cancel)
+            self._polling_tasks[ck].clear()
+            self._refresh_map[ck].clear()
+        return

deepeval/test_run/__init__.py

@@ -11,7 +11,7 @@ from .test_run import (
 )
 
 from .hooks import on_test_run_end, invoke_test_run_end_hook
-from .api import MetricData
+from .api import MetricData, TurnApi
 from .hyperparameters import log_hyperparameters
 
 
@@ -28,5 +28,6 @@ __all__ = [
     "on_test_run_end",
     "invoke_test_run_end_hook",
     "MetricData",
+    "TurnApi",
     "log_hyperparameters",
 ]

deepeval/test_run/api.py

@@ -99,6 +99,7 @@ class TurnApi(BaseModel):
     role: str
     content: str
     order: int
+    user_id: Optional[str] = Field(None, alias="userId")
     retrieval_context: Optional[list] = Field(None, alias="retrievalContext")
     tools_called: Optional[List[ToolCall]] = Field(None, alias="toolsCalled")
     additional_metadata: Optional[Dict] = Field(

deepeval/test_run/test_run.py

@@ -2,9 +2,8 @@ from enum import Enum
 import os
 import json
 from pydantic import BaseModel, Field
-from typing import Any, Optional, List, Dict, Union
+from typing import Any, Optional, List, Dict, Union, Tuple
 import shutil
-import webbrowser
 import sys
 import datetime
 import portalocker
@@ -27,6 +26,9 @@ from deepeval.utils import (
     delete_file_if_exists,
     get_is_running_deepeval,
     open_browser,
+    shorten,
+    format_turn,
+    len_short,
 )
 from deepeval.test_run.cache import global_test_run_cache_manager
 from deepeval.constants import CONFIDENT_TEST_CASE_BATCH_SIZE, HIDDEN_DIR
@@ -546,7 +548,7 @@ class TestRunManager:
 
             if (
                 display == TestRunResultDisplay.PASSING
-                and test_case.success == False
+                and test_case.success is False
             ):
                 continue
             elif display == TestRunResultDisplay.FAILING and test_case.success:
@@ -618,7 +620,7 @@ class TestRunManager:
         ):
             if (
                 display == TestRunResultDisplay.PASSING
-                and conversational_test_case.success == False
+                and conversational_test_case.success is False
             ):
                 continue
             elif (
@@ -631,6 +633,65 @@ class TestRunManager:
             fail_count = 0
             conversational_test_case_name = conversational_test_case.name
 
+            if conversational_test_case.turns:
+                turns_table = Table(
+                    title=f"Conversation - {conversational_test_case_name}",
+                    show_header=True,
+                    header_style="bold",
+                )
+                turns_table.add_column("#", justify="right", width=3)
+                turns_table.add_column("Role", justify="left", width=10)
+
+                # subtract fixed widths + borders and padding.
+                # ~20 as a safe buffer
+                details_max_width = max(
+                    48, min(120, console.width - 3 - 10 - 20)
+                )
+                turns_table.add_column(
+                    "Details",
+                    justify="left",
+                    overflow="fold",
+                    max_width=details_max_width,
+                )
+
+                # truncate when too long
+                tools_max_width = min(60, max(24, console.width // 3))
+                turns_table.add_column(
+                    "Tools",
+                    justify="left",
+                    no_wrap=True,
+                    overflow="ellipsis",
+                    max_width=tools_max_width,
+                )
+
+                sorted_turns = sorted(
+                    conversational_test_case.turns, key=lambda t: t.order
+                )
+
+                for t in sorted_turns:
+                    tools = t.tools_called or []
+                    tool_names = ", ".join(tc.name for tc in tools)
+
+                    # omit order, role and tools since we show them in a separate columns.
+                    details = format_turn(
+                        t,
+                        include_tools_in_header=False,
+                        include_order_role_in_header=False,
+                    )
+
+                    turns_table.add_row(
+                        str(t.order),
+                        t.role,
+                        details,
+                        shorten(tool_names, len_short()),
+                    )
+
+                console.print(turns_table)
+            else:
+                console.print(
+                    f"[dim]No turns recorded for {conversational_test_case_name}.[/dim]"
+                )
+
             if conversational_test_case.metrics_data is not None:
                 for metric_data in conversational_test_case.metrics_data:
                     if metric_data.success:
@@ -698,7 +759,7 @@ class TestRunManager:
         )
         print(table)
 
-    def post_test_run(self, test_run: TestRun) -> Optional[str]:
+    def post_test_run(self, test_run: TestRun) -> Optional[Tuple[str, str]]:
         if (
             len(test_run.test_cases) == 0
             and len(test_run.conversational_test_cases) == 0
@@ -752,6 +813,21 @@ class TestRunManager:
             body=body,
         )
 
+        if not isinstance(data, dict) or "id" not in data:
+            # try to show helpful details
+            detail = None
+            if isinstance(data, dict):
+                detail = (
+                    data.get("detail")
+                    or data.get("message")
+                    or data.get("error")
+                )
+            # fall back to repr for visibility
+            raise RuntimeError(
+                f"Confident API response missing 'id'. "
+                f"detail={detail!r} raw={type(data).__name__}:{repr(data)[:500]}"
+            )
+
         res = TestRunHttpResponse(
             id=data["id"],
         )
@@ -814,7 +890,7 @@ class TestRunManager:
         )
         self.save_final_test_run_link(link)
         open_browser(link)
-        return link
+        return link, res.id
 
     def save_test_run_locally(self):
         local_folder = os.getenv("DEEPEVAL_RESULTS_FOLDER")
@@ -841,7 +917,7 @@ class TestRunManager:
         runDuration: float,
         display_table: bool = True,
         display: Optional[TestRunResultDisplay] = TestRunResultDisplay.ALL,
-    ) -> Optional[str]:
+    ) -> Optional[Tuple[str, str]]:
         test_run = self.get_test_run()
         if test_run is None:
             print("Test Run is empty, please try again.")
@@ -868,8 +944,8 @@ class TestRunManager:
         test_run.sort_test_cases()
 
         if global_test_run_cache_manager.disable_write_cache is None:
-            global_test_run_cache_manager.disable_write_cache = (
-                get_is_running_deepeval() == False
+            global_test_run_cache_manager.disable_write_cache = not bool(
+                get_is_running_deepeval()
             )
 
         global_test_run_cache_manager.wrap_up_cached_test_run()

deepeval/tracing/__init__.py

@@ -4,6 +4,7 @@ from .context import (
     update_retriever_span,
     update_llm_span,
 )
+from .trace_context import trace
 from .types import BaseSpan, Trace
 from .tracing import observe, trace_manager
 from .offline_evals import evaluate_thread, evaluate_trace, evaluate_span
@@ -16,6 +17,7 @@ __all__ = [
     "BaseSpan",
     "Trace",
     "observe",
+    "trace",
     "trace_manager",
     "evaluate_thread",
     "evaluate_trace",