deepeval 3.8.1__py3-none-any.whl → 3.8.3__py3-none-any.whl

deepeval/integrations/langchain/utils.py

@@ -1,5 +1,77 @@
-from typing import Any, List, Dict, Optional
+import uuid
+from typing import Any, List, Dict, Optional, Union, Literal, Callable
+from time import perf_counter
 from langchain_core.outputs import ChatGeneration
+from rich.progress import Progress
+
+from deepeval.metrics import BaseMetric
+from deepeval.tracing.context import current_span_context, current_trace_context
+from deepeval.tracing.tracing import trace_manager
+from deepeval.tracing.types import (
+    AgentSpan,
+    BaseSpan,
+    LlmSpan,
+    RetrieverSpan,
+    SpanType,
+    ToolSpan,
+    TraceSpanStatus,
+)
+
+
+def convert_chat_messages_to_input(
+    messages: list[list[Any]], **kwargs
+) -> List[Dict[str, str]]:
+    """
+    Convert LangChain chat messages to our internal format.
+
+    Args:
+        messages: list[list[BaseMessage]] - outer list is batches, inner is messages.
+        **kwargs: May contain invocation_params with tools definitions.
+
+    Returns:
+        List of dicts with 'role' and 'content' keys, matching the schema used
+        by parse_prompts_to_messages for consistency.
+    """
+    # Valid roles matching parse_prompts_to_messages
+    ROLE_MAPPING = {
+        "human": "human",
+        "user": "human",
+        "ai": "ai",
+        "assistant": "ai",
+        "system": "system",
+        "tool": "tool",
+        "function": "function",
+    }
+
+    result: List[Dict[str, str]] = []
+    for batch in messages:
+        for msg in batch:
+            # BaseMessage has .type (role) and .content
+            raw_role = getattr(msg, "type", "unknown")
+            content = getattr(msg, "content", "")
+
+            # Normalize role using same conventions as prompt parsing
+            role = ROLE_MAPPING.get(raw_role.lower(), raw_role)
+
+            # Convert content to string (handles empty content, lists, etc.)
+            if isinstance(content, list):
+                # Some messages have content as a list of content blocks
+                content_str = " ".join(
+                    str(c.get("text", c) if isinstance(c, dict) else c)
+                    for c in content
+                )
+            else:
+                content_str = str(content) if content else ""
+
+            result.append({"role": role, "content": content_str})
+
+    # Append tool definitions if present which matches parse_prompts_to_messages behavior
+    tools = kwargs.get("invocation_params", {}).get("tools", None)
+    if tools and isinstance(tools, list):
+        for tool in tools:
+            result.append({"role": "Tool Input", "content": str(tool)})
+
+    return result
 
 
 def parse_prompts_to_messages(
@@ -112,27 +184,6 @@ def safe_extract_model_name(
     return None
 
 
-from typing import Any, List, Dict, Optional, Union, Literal, Callable
-from langchain_core.outputs import ChatGeneration
-from time import perf_counter
-import uuid
-from rich.progress import Progress
-from deepeval.tracing.tracing import Observer
-
-from deepeval.metrics import BaseMetric
-from deepeval.tracing.context import current_span_context, current_trace_context
-from deepeval.tracing.tracing import trace_manager
-from deepeval.tracing.types import (
-    AgentSpan,
-    BaseSpan,
-    LlmSpan,
-    RetrieverSpan,
-    SpanType,
-    ToolSpan,
-    TraceSpanStatus,
-)
-
-
 def enter_current_context(
     span_type: Optional[
         Union[Literal["agent", "llm", "retriever", "tool"], str]
@@ -239,8 +290,8 @@ def enter_current_context(
 
     if (
         parent_span
-        and getattr(parent_span, "progress", None) is not None
-        and getattr(parent_span, "pbar_callback_id", None) is not None
+        and parent_span.progress is not None
+        and parent_span.pbar_callback_id is not None
     ):
         progress = parent_span.progress
         pbar_callback_id = parent_span.pbar_callback_id

deepeval/integrations/llama_index/handler.py

@@ -21,6 +21,7 @@ from deepeval.tracing.types import (
 from deepeval.tracing.trace_context import (
     current_llm_context,
     current_agent_context,
+    current_trace_context,
 )
 from deepeval.test_case import ToolCall
 from deepeval.tracing.utils import make_json_serializable
@@ -40,7 +41,10 @@ try:
         LLMChatStartEvent,
         LLMChatEndEvent,
     )
-    from llama_index_instrumentation.dispatcher import Dispatcher
+    from llama_index.core.instrumentation import Dispatcher
+    from llama_index.core.instrumentation.events.retrieval import (
+        RetrievalEndEvent,
+    )
     from deepeval.integrations.llama_index.utils import (
         parse_id,
         prepare_input_llm_test_case_params,
@@ -82,15 +86,23 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 input_messages.append({"role": role, "content": content})
 
             llm_span_context = current_llm_context.get()
-            # create the span
+
+            parent_span = trace_manager.get_span_by_uuid(event.span_id)
+            if parent_span:
+                trace_uuid = parent_span.trace_uuid
+            else:
+                current_trace = current_trace_context.get()
+                if current_trace:
+                    trace_uuid = current_trace.uuid
+                else:
+                    trace_uuid = trace_manager.start_new_trace().uuid
+
             llm_span = LlmSpan(
                 name="ConfidentLLMSpan",
                 uuid=str(uuid.uuid4()),
                 status=TraceSpanStatus.IN_PROGRESS,
                 children=[],
-                trace_uuid=trace_manager.get_span_by_uuid(
-                    event.span_id
-                ).trace_uuid,
+                trace_uuid=trace_uuid,
                 parent_uuid=event.span_id,
                 start_time=perf_counter(),
                 model=getattr(event, "model_dict", {}).get(
@@ -128,6 +140,13 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     trace_manager.remove_span(llm_span.uuid)
                     del self.open_ai_astream_to_llm_span_map[event.span_id]
 
+        if isinstance(event, RetrievalEndEvent):
+            span = trace_manager.get_span_by_uuid(event.span_id)
+            if span:
+                span.retrieval_context = [
+                    node.node.get_content() for node in event.nodes
+                ]
+
     def new_span(
         self,
         id_: str,
@@ -139,18 +158,30 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
     ) -> Optional[LlamaIndexBaseSpan]:
         class_name, method_name = parse_id(id_)
 
-        # check if it is a root span
-        if parent_span_id is None:
-            trace_uuid = trace_manager.start_new_trace().uuid
-        elif class_name == "Workflow" and method_name == "run":
-            trace_uuid = trace_manager.start_new_trace().uuid
-            parent_span_id = None  # since workflow is the root span, we need to set the parent span id to None
+        current_trace = current_trace_context.get()
+        trace_uuid = None
+
+        if parent_span_id is None or (
+            class_name == "Workflow" and method_name == "run"
+        ):
+            if current_trace:
+                trace_uuid = current_trace.uuid
+            else:
+                trace_uuid = trace_manager.start_new_trace().uuid
+
+            if class_name == "Workflow" and method_name == "run":
+                parent_span_id = None
+
         elif trace_manager.get_span_by_uuid(parent_span_id):
             trace_uuid = trace_manager.get_span_by_uuid(
                 parent_span_id
             ).trace_uuid
+
         else:
-            trace_uuid = trace_manager.start_new_trace().uuid
+            if current_trace:
+                trace_uuid = current_trace.uuid
+            else:
+                trace_uuid = trace_manager.start_new_trace().uuid
 
         self.root_span_trace_id_map[id_] = trace_uuid
 
@@ -195,7 +226,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     else None
                 ),
             )
-        elif method_name == "acall":
+        elif method_name in ["acall", "call_tool", "acall_tool"]:
             span = ToolSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -206,7 +237,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 input=bound_args.arguments,
                 name="Tool",
             )
-        # prepare input test case params for the span
+
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments
         )
@@ -215,6 +246,22 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         return span
 
+    def _get_output_value(self, result: Any) -> Any:
+        """Helper to ensure AgentChatResponse and similar objects are serialized as dicts."""
+        if hasattr(result, "response") and hasattr(result, "sources"):
+            if hasattr(result, "model_dump"):
+                return result.model_dump()
+            if hasattr(result, "to_dict"):
+                return result.to_dict()
+            return {"response": result.response, "sources": result.sources}
+
+        if hasattr(result, "response"):
+            if hasattr(result, "model_dump"):
+                return result.model_dump()
+            return {"response": result.response}
+
+        return result
+
     def prepare_to_exit_span(
         self,
         id_: str,
@@ -229,7 +276,8 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
             return None
 
         class_name, method_name = parse_id(id_)
-        if method_name == "call_tool":
+
+        if method_name in ["call_tool", "acall_tool"]:
             output_json = make_json_serializable(result)
             if output_json and isinstance(output_json, dict):
                 if base_span.tools_called is None:
@@ -243,7 +291,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 )
         base_span.end_time = perf_counter()
         base_span.status = TraceSpanStatus.SUCCESS
-        base_span.output = result
+        base_span.output = self._get_output_value(result)
 
         if isinstance(base_span, ToolSpan):
             result_json = make_json_serializable(result)
@@ -265,7 +313,8 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         if base_span.parent_uuid is None:
             trace_manager.end_trace(base_span.trace_uuid)
-            self.root_span_trace_id_map.pop(base_span.uuid)
+            if base_span.uuid in self.root_span_trace_id_map:
+                self.root_span_trace_id_map.pop(base_span.uuid)
 
         return base_span
 
@@ -282,13 +331,12 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
             return None
 
         base_span.end_time = perf_counter()
-        base_span.status = (
-            TraceSpanStatus.SUCCESS
-        )  # find a way to add error and handle the span without the parent id
+        base_span.status = TraceSpanStatus.SUCCESS
 
         if base_span.parent_uuid is None:
             trace_manager.end_trace(base_span.trace_uuid)
-            self.root_span_trace_id_map.pop(base_span.uuid)
+            if base_span.uuid in self.root_span_trace_id_map:
+                self.root_span_trace_id_map.pop(base_span.uuid)
 
         return base_span
 

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -36,10 +36,14 @@ try:
         SpanProcessor as _SpanProcessor,
         TracerProvider,
     )
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace.export import (
+        BatchSpanProcessor,
+        SimpleSpanProcessor,
+    )
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
+    from opentelemetry.trace import set_tracer_provider
     from pydantic_ai.models.instrumented import (
         InstrumentationSettings as _BaseInstrumentationSettings,
     )
@@ -131,7 +135,12 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
     ):
         is_dependency_installed()
 
-        _environment = os.getenv("CONFIDENT_TRACE_ENVIRONMENT", "development")
+        if trace_manager.environment is not None:
+            _environment = trace_manager.environment
+        elif settings.CONFIDENT_TRACE_ENVIRONMENT is not None:
+            _environment = settings.CONFIDENT_TRACE_ENVIRONMENT
+        else:
+            _environment = "development"
         if _environment and _environment in [
             "production",
             "staging",
@@ -166,7 +175,9 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         trace_provider.add_span_processor(span_interceptor)
 
         if is_test_mode:
-            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+            trace_provider.add_span_processor(
+                SimpleSpanProcessor(ConfidentSpanExporter())
+            )
         else:
             trace_provider.add_span_processor(
                 BatchSpanProcessor(
@@ -176,6 +187,12 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
                     )
                 )
             )
+        try:
+            set_tracer_provider(trace_provider)
+        except Exception as e:
+            # Handle case where provider is already set (optional warning)
+            logger.warning(f"Could not set global tracer provider: {e}")
+
         super().__init__(tracer_provider=trace_provider)
 
 
@@ -234,16 +251,14 @@ class SpanInterceptor(SpanProcessor):
             )
 
         # set agent name and metric collection
-        if span.attributes.get("agent_name"):
-            span.set_attribute("confident.span.type", "agent")
-            span.set_attribute(
-                "confident.span.name", span.attributes.get("agent_name")
-            )
-            if self.settings.agent_metric_collection:
-                span.set_attribute(
-                    "confident.span.metric_collection",
-                    self.settings.agent_metric_collection,
-                )
+        agent_name = (
+            span.attributes.get("gen_ai.agent.name")
+            or span.attributes.get("pydantic_ai.agent.name")
+            or span.attributes.get("agent_name")
+        )
+
+        if agent_name:
+            self._add_agent_span(span, agent_name)
 
         # set llm metric collection
         if span.attributes.get("gen_ai.operation.name") in [
@@ -270,6 +285,19 @@ class SpanInterceptor(SpanProcessor):
                 )
 
     def on_end(self, span):
+
+        already_processed = (
+            span.attributes.get("confident.span.type") == "agent"
+        )
+        if not already_processed:
+            agent_name = (
+                span.attributes.get("gen_ai.agent.name")
+                or span.attributes.get("pydantic_ai.agent.name")
+                or span.attributes.get("agent_name")
+            )
+            if agent_name:
+                self._add_agent_span(span, agent_name)
+
         if self.settings.is_test_mode:
             if span.attributes.get("confident.span.type") == "agent":
 
@@ -322,4 +350,12 @@ class SpanInterceptor(SpanProcessor):
                 trace.status = TraceSpanStatus.SUCCESS
                 trace.end_time = perf_counter()
                 trace_manager.traces_to_evaluate.append(trace)
-                test_exporter.clear_span_json_list()
+
+    def _add_agent_span(self, span, name):
+        span.set_attribute("confident.span.type", "agent")
+        span.set_attribute("confident.span.name", name)
+        if self.settings.agent_metric_collection:
+            span.set_attribute(
+                "confident.span.metric_collection",
+                self.settings.agent_metric_collection,
+            )

deepeval/integrations/pydantic_ai/otel.py

@@ -2,6 +2,7 @@ import warnings
 from typing import Optional
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.config.settings import get_settings
+import logging
 
 try:
     from opentelemetry import trace
@@ -24,6 +25,9 @@ def is_opentelemetry_available():
     return True
 
 
+logger = logging.getLogger(__name__)
+settings = get_settings()
+
 settings = get_settings()
 # OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
 
@@ -51,6 +55,11 @@ def instrument_pydantic_ai(api_key: Optional[str] = None):
                 )
             )
         )
+        try:
+            trace.set_tracer_provider(tracer_provider)
+        except Exception as e:
+            # Handle case where provider is already set (optional warning)
+            logger.warning(f"Could not set global tracer provider: {e}")
 
         # create an instrumented exporter
         from pydantic_ai.models.instrumented import InstrumentationSettings

deepeval/metrics/utils.py

@@ -320,6 +320,17 @@ def check_llm_test_case_params(
         metric.error = error_str
         raise ValueError(error_str)
 
+    # Centralized: if a metric requires actual_output, reject empty/whitespace
+    # (including empty multimodal outputs) as "missing params".
+    if LLMTestCaseParams.ACTUAL_OUTPUT in test_case_params:
+        actual_output = getattr(
+            test_case, LLMTestCaseParams.ACTUAL_OUTPUT.value
+        )
+        if isinstance(actual_output, str) and actual_output == "":
+            error_str = f"'actual_output' cannot be empty for the '{metric.__name__}' metric"
+            metric.error = error_str
+            raise MissingTestCaseParamsError(error_str)
+
     missing_params = []
     for param in test_case_params:
         if getattr(test_case, param.value) is None:

deepeval/simulator/conversation_simulator.py

@@ -610,7 +610,8 @@ class ConversationSimulator:
     ) -> BaseModel:
         if self.using_native_model:
             res, cost = self.simulator_model.generate(prompt, schema=schema)
-            self.simulation_cost += cost
+            if cost is not None:
+                self.simulation_cost += cost
             return res
         else:
             try:
@@ -630,7 +631,8 @@ class ConversationSimulator:
             res, cost = await self.simulator_model.a_generate(
                 prompt, schema=schema
             )
-            self.simulation_cost += cost
+            if cost is not None:
+                self.simulation_cost += cost
             return res
         else:
             try:

deepeval/telemetry.py

@@ -1,5 +1,4 @@
 from contextlib import contextmanager
-import logging
 import os
 import socket
 import sys
@@ -85,13 +84,6 @@ if not telemetry_opt_out():
 anonymous_public_ip = None
 
 if not telemetry_opt_out():
-    from opentelemetry import trace
-    from opentelemetry.sdk.trace import TracerProvider
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
-    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
-        OTLPSpanExporter,
-    )
-
     anonymous_public_ip = get_anonymous_public_ip()
     sentry_sdk.init(
         dsn="https://5ef587d58109ee45d6544f3657efdd1f@o4506098477236224.ingest.sentry.io/4506098479136768",
@@ -102,27 +94,6 @@ if not telemetry_opt_out():
         default_integrations=False,  # Disable Sentry's default integrations
     )
 
-    # Set up the Tracer Provider
-    trace.set_tracer_provider(TracerProvider())
-    tracer_provider = trace.get_tracer_provider()
-
-    # New Relic License Key and OTLP Endpoint
-    NEW_RELIC_LICENSE_KEY = "1711c684db8a30361a7edb0d0398772cFFFFNRAL"
-    NEW_RELIC_OTLP_ENDPOINT = "https://otlp.nr-data.net:4317"
-    otlp_exporter = OTLPSpanExporter(
-        endpoint=NEW_RELIC_OTLP_ENDPOINT,
-        headers={"api-key": NEW_RELIC_LICENSE_KEY},
-    )
-
-    # Add the OTLP exporter to the span processor
-    span_processor = BatchSpanProcessor(otlp_exporter)
-    tracer_provider.add_span_processor(span_processor)
-
-    logging.getLogger("opentelemetry.exporter.otlp").setLevel(logging.CRITICAL)
-
-    # Create a tracer for your application
-    tracer = trace.get_tracer(__name__)
-
     # Initialize PostHog
     posthog = Posthog(
         project_api_key="phc_IXvGRcscJJoIb049PtjIZ65JnXQguOUZ5B5MncunFdB",
@@ -199,11 +170,7 @@ def capture_evaluation_run(type: str):
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -227,11 +194,7 @@ def capture_recommend_metrics():
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -259,11 +222,7 @@ def capture_metric_type(
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -297,11 +256,7 @@ def capture_synthesizer_run(
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -330,11 +285,7 @@ def capture_conversation_simulator_run(num_conversations: int):
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -360,11 +311,7 @@ def capture_guardrails(guards: List[str]):
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -391,11 +338,7 @@ def capture_benchmark_run(benchmark: str, num_tasks: int):
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -421,11 +364,7 @@ def capture_login_event():
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -451,11 +390,7 @@ def capture_view_event():
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 @contextmanager
@@ -478,11 +413,7 @@ def capture_pull_dataset():
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 # track metrics that are components and metrics that aren't components
@@ -509,11 +440,7 @@ def capture_send_trace():
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            yield span
+        yield
 
 
 # tracing integration
@@ -542,13 +469,7 @@ def capture_tracing_integration(integration_name: str):
         posthog.capture(
             distinct_id=distinct_id, event=event, properties=properties
         )
-        # capture new relic
-        with tracer.start_as_current_span(event) as span:
-            for property, value in properties.items():
-                span.set_attribute(property, value)
-            # OTEL/New Relic filtering attributes
-            span.set_attribute("integration.name", integration_name)
-            yield span
+        yield
 
 
 #########################################################

deepeval/tracing/api.py

@@ -126,6 +126,7 @@ class TraceApi(BaseModel):
     input: Optional[Any] = Field(None)
     output: Optional[Any] = Field(None)
     status: Optional[TraceSpanApiStatus] = Field(TraceSpanApiStatus.SUCCESS)
+    test_case_id: Optional[str] = Field(None, alias="testCaseId")
 
     # additional test case parameters
     retrieval_context: Optional[List[str]] = Field(