deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/tracing/otel/exporter.py

@@ -254,7 +254,7 @@ class ConfidentSpanExporter(SpanExporter):
 
         base_span = None
         try:
-            base_span = self.__prepare_boilerplate_base_span(span)
+            base_span = self.prepare_boilerplate_base_span(span)
         except Exception:
             pass
 
@@ -453,9 +453,8 @@ class ConfidentSpanExporter(SpanExporter):
         if span_output:
             base_span.output = span_output
 
-    def __prepare_boilerplate_base_span(
-        self, span: ReadableSpan
-    ) -> Optional[BaseSpan]:
+    @staticmethod
+    def prepare_boilerplate_base_span(span: ReadableSpan) -> Optional[BaseSpan]:
 
         ################ Get Span Type ################
         span_type = span.attributes.get("confident.span.type")

deepeval/tracing/otel/utils.py

@@ -3,7 +3,7 @@ import json
 from typing import List, Optional, Tuple, Any
 from opentelemetry.sdk.trace.export import ReadableSpan
 
-from deepeval.evaluate.utils import create_api_test_case
+from deepeval.test_case.api import create_api_test_case
 from deepeval.test_run.api import LLMApiTestCase
 from deepeval.test_run.test_run import global_test_run_manager
 from deepeval.tracing.types import Trace, LLMTestCase, ToolCall
@@ -11,7 +11,7 @@ from deepeval.tracing import trace_manager, BaseSpan
 from deepeval.tracing.utils import make_json_serializable
 
 
-GEN_AI_OPERATION_NAMES = ["chat", "generate_content", "task_completion"]
+GEN_AI_OPERATION_NAMES = ["chat", "generate_content", "text_completion"]
 
 
 def to_hex_string(id_value: int | bytes, length: int = 32) -> str:
@@ -128,6 +128,10 @@ def check_llm_input_from_gen_ai_attributes(
 
         input = system_instructions + input_messages
 
+        model_parameters = check_model_parameters(span)
+        if model_parameters:
+            input.append(model_parameters)
+
     except Exception:
         pass
     try:
@@ -413,7 +417,7 @@ def post_test_run(traces: List[Trace], test_run_id: Optional[str]):
     # return test_run_manager.post_test_run(test_run) TODO: add after test run with metric collection is implemented
 
 
-def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
+def normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     try:
         raw = span.attributes.get("pydantic_ai.all_messages")
         if not raw:
@@ -438,7 +442,7 @@ def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     except Exception:
         pass
 
-    return None
+    return []
 
 
 def _extract_non_thinking_part_of_last_message(message: dict) -> dict:
@@ -461,7 +465,7 @@ def check_pydantic_ai_agent_input_output(
     output_val: Optional[Any] = None
 
     # Get normalized messages once
-    normalized = _normalize_pydantic_ai_messages(span)
+    normalized = normalize_pydantic_ai_messages(span)
 
     # Input (pydantic_ai.all_messages) - slice up to and including the first 'user' message
     if normalized:
@@ -523,3 +527,18 @@ def check_pydantic_ai_trace_input_output(
         input_val, output_val = check_pydantic_ai_agent_input_output(span)
 
     return input_val, output_val
+
+
+def check_model_parameters(span: ReadableSpan) -> Optional[dict]:
+    try:
+        raw_model_parameters = span.attributes.get("model_request_parameters")
+        if raw_model_parameters and isinstance(raw_model_parameters, str):
+            model_parameters = json.loads(raw_model_parameters)
+            if isinstance(model_parameters, dict):
+                return {
+                    "role": "Model Request Parameters",
+                    "content": model_parameters,
+                }
+    except Exception:
+        pass
+    return None

deepeval/tracing/trace_context.py

@@ -1,14 +1,98 @@
-from .context import current_trace_context
-from .tracing import trace_manager
+from typing import Optional, List, Dict, Any
+from contextvars import ContextVar
 from contextlib import contextmanager
+from dataclasses import dataclass
+
+from .tracing import trace_manager
+from .context import current_trace_context, update_current_trace
+from deepeval.prompt import Prompt
+from deepeval.metrics import BaseMetric
+from deepeval.test_case.llm_test_case import ToolCall
+
+
+@dataclass
+class LlmSpanContext:
+    prompt: Optional[Prompt] = None
+    metrics: Optional[List[BaseMetric]] = None
+    metric_collection: Optional[str] = None
+    expected_output: Optional[str] = None
+    expected_tools: Optional[List[ToolCall]] = None
+    context: Optional[List[str]] = None
+    retrieval_context: Optional[List[str]] = None
+
+
+@dataclass
+class AgentSpanContext:
+    metrics: Optional[List[BaseMetric]] = None
+    metric_collection: Optional[str] = None
+    expected_output: Optional[str] = None
+    expected_tools: Optional[List[ToolCall]] = None
+    context: Optional[List[str]] = None
+    retrieval_context: Optional[List[str]] = None
+
+
+current_llm_context: ContextVar[Optional[LlmSpanContext]] = ContextVar(
+    "current_llm_context", default=LlmSpanContext()
+)
+
+current_agent_context: ContextVar[Optional[AgentSpanContext]] = ContextVar(
+    "current_agent_context", default=AgentSpanContext()
+)
 
 
 @contextmanager
-def trace():
+def trace(
+    llm_span_context: Optional[LlmSpanContext] = None,
+    agent_span_context: Optional[AgentSpanContext] = None,
+    name: Optional[str] = None,
+    tags: Optional[List[str]] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+    thread_id: Optional[str] = None,
+    user_id: Optional[str] = None,
+    input: Optional[Any] = None,
+    output: Optional[Any] = None,
+    retrieval_context: Optional[List[str]] = None,
+    context: Optional[List[str]] = None,
+    expected_output: Optional[str] = None,
+    tools_called: Optional[List[ToolCall]] = None,
+    expected_tools: Optional[List[ToolCall]] = None,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
     current_trace = current_trace_context.get()
 
     if not current_trace:
         current_trace = trace_manager.start_new_trace()
-        current_trace_context.set(current_trace)
 
-    yield current_trace
+    if metrics:
+        current_trace.metrics = metrics
+
+    if metric_collection:
+        current_trace.metric_collection = metric_collection
+
+    current_trace_context.set(current_trace)
+
+    update_current_trace(
+        name=name,
+        tags=tags,
+        metadata=metadata,
+        thread_id=thread_id,
+        user_id=user_id,
+        input=input,
+        output=output,
+        retrieval_context=retrieval_context,
+        context=context,
+        expected_output=expected_output,
+        tools_called=tools_called,
+        expected_tools=expected_tools,
+    )
+
+    if llm_span_context:
+        current_llm_context.set(llm_span_context)
+    if agent_span_context:
+        current_agent_context.set(agent_span_context)
+    try:
+        yield current_trace
+    finally:
+        current_llm_context.set(LlmSpanContext())
+        current_agent_context.set(AgentSpanContext())

deepeval/tracing/tracing.py

@@ -1,5 +1,15 @@
-import os
-from typing import Any, Dict, List, Literal, Optional, Set, Union, Callable
+import weakref
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Union,
+)
 from time import perf_counter
 import threading
 import functools
@@ -20,6 +30,7 @@ from deepeval.constants import (
 )
 from deepeval.confident.api import Api, Endpoints, HttpMethods, is_confident
 from deepeval.metrics import BaseMetric
+from deepeval.test_case.llm_test_case import ToolCall
 from deepeval.tracing.api import (
     BaseApiSpan,
     SpanApiType,
@@ -41,6 +52,7 @@ from deepeval.tracing.types import (
 )
 from deepeval.tracing.utils import (
     Environment,
+    prepare_tool_call_input_parameters,
     replace_self_with_class_name,
     make_json_serializable,
     perf_counter_to_datetime,
@@ -55,6 +67,10 @@ from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.tracing.api import PromptApi
 from deepeval.tracing.trace_test_manager import trace_testing_manager
 
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden
+
 EVAL_DUMMY_SPAN_NAME = "evals_iterator"
 
 
@@ -65,6 +81,10 @@ class TraceManager:
         self.active_spans: Dict[str, BaseSpan] = (
             {}
         )  # Map of span_uuid to BaseSpan
+        # Map each trace created during evaluation_loop to the Golden that was active
+        # when it was started. This lets us evaluate traces against the correct golden
+        # since we cannot rely on positional indexing as the order is not guaranteed.
+        self.trace_uuid_to_golden: Dict[str, Golden] = {}
 
         settings = get_settings()
         # Initialize queue and worker thread for trace posting
@@ -73,6 +93,9 @@ class TraceManager:
         self._min_interval = 0.2  # Minimum time between API calls (seconds)
         self._last_post_time = 0
         self._in_flight_tasks: Set[asyncio.Task[Any]] = set()
+        self.task_bindings: "weakref.WeakKeyDictionary[asyncio.Task, dict]" = (
+            weakref.WeakKeyDictionary()
+        )
         self._flush_enabled = bool(settings.CONFIDENT_TRACE_FLUSH)
         self._daemon = not self._flush_enabled
 
@@ -86,7 +109,7 @@ class TraceManager:
         )
         validate_environment(self.environment)
 
-        self.sampling_rate = settings.CONFIDENT_SAMPLE_RATE
+        self.sampling_rate = settings.CONFIDENT_TRACE_SAMPLE_RATE
         validate_sampling_rate(self.sampling_rate)
         self.openai_client = None
         self.tracing_enabled = True
@@ -166,6 +189,19 @@ class TraceManager:
         self.traces.append(new_trace)
         if self.evaluation_loop:
             self.traces_to_evaluate_order.append(trace_uuid)
+            # Associate the current Golden with this trace so we can
+            # later evaluate traces against the correct golden, even if more traces
+            # are created than goldens or the order interleaves.
+            try:
+                from deepeval.contextvars import get_current_golden
+
+                current_golden = get_current_golden()
+                if current_golden is not None:
+                    self.trace_uuid_to_golden[trace_uuid] = current_golden
+            except Exception:
+                # not much we can do, but if the golden is not there during evaluation
+                # we will write out a verbose debug log
+                pass
         return new_trace
 
     def end_trace(self, trace_uuid: str):
@@ -820,6 +856,25 @@ class Observer:
             self._progress = parent_span.progress
             self._pbar_callback_id = parent_span.pbar_callback_id
 
+        try:
+            import asyncio
+
+            task = asyncio.current_task()
+        except Exception:
+            task = None
+
+        if task is not None:
+            binding = trace_manager.task_bindings.get(task) or {}
+            # record the trace the task is working on
+            binding["trace_uuid"] = span_instance.trace_uuid
+            # only set root_span_uuid when this span is a root. Don't do this for child or we will override our record.
+            if (
+                span_instance.parent_uuid is None
+                and "root_span_uuid" not in binding
+            ):
+                binding["root_span_uuid"] = span_instance.uuid
+            trace_manager.task_bindings[task] = binding
+
         if self._progress is not None and self._pbar_callback_id is not None:
             span_instance.progress = self._progress
             span_instance.pbar_callback_id = self._pbar_callback_id
@@ -861,6 +916,22 @@ class Observer:
         ):
             current_span.prompt = self.prompt
 
+        if not current_span.tools_called:
+            # check any tool span children
+            for child in current_span.children:
+                if isinstance(child, ToolSpan):
+                    current_span.tools_called = current_span.tools_called or []
+                    current_span.tools_called.append(
+                        ToolCall(
+                            name=child.name,
+                            description=child.description,
+                            input_parameters=prepare_tool_call_input_parameters(
+                                child.input
+                            ),
+                            output=child.output,
+                        )
+                    )
+
         trace_manager.remove_span(self.uuid)
         if current_span.parent_uuid:
             parent_span = trace_manager.get_span_by_uuid(

deepeval/tracing/types.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from dataclasses import dataclass, field
 from pydantic import BaseModel, Field
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Literal
 from rich.progress import Progress
 
 from deepeval.prompt.prompt import Prompt
@@ -10,6 +10,19 @@ from deepeval.test_case import LLMTestCase
 from deepeval.metrics import BaseMetric
 
 
+class Message(BaseModel):
+    role: str
+    """To be displayed on the top of the message block."""
+
+    type: Literal["tool_calls", "tool_output", "thinking", "default"] = (
+        "default"
+    )
+    """Decides how the content is rendered."""
+
+    content: Any
+    """The content of the message."""
+
+
 class TraceWorkerStatus(Enum):
     SUCCESS = "success"
     FAILURE = "failure"
@@ -44,7 +57,7 @@ class LlmOutput(BaseModel):
 class BaseSpan(BaseModel):
     uuid: str
     status: TraceSpanStatus
-    children: List["BaseSpan"]
+    children: List["BaseSpan"] = Field(default_factory=list)
     trace_uuid: str = Field(serialization_alias="traceUuid")
     parent_uuid: Optional[str] = Field(None, serialization_alias="parentUuid")
     start_time: float = Field(serialization_alias="startTime")
@@ -88,6 +101,7 @@ class AgentSpan(BaseSpan):
 
 
 class LlmSpan(BaseSpan):
+
     model: Optional[str] = None
     prompt: Optional[Prompt] = None
     input_token_count: Optional[float] = Field(
@@ -106,6 +120,10 @@ class LlmSpan(BaseSpan):
         None, serialization_alias="tokenTimes"
     )
 
+    # input_tools: Optional[List[ToolSchema]] = Field(None, serialization_alias="inputTools")
+    # invocation_params: Optional[Dict[str, Any]] = Field(None, serialization_alias="invocationParams")
+    # output_metadata: Optional[Dict[str, Any]] = Field(None, serialization_alias="outputMetadata")
+
     # for serializing `prompt`
     model_config = {"arbitrary_types_allowed": True}
 

deepeval/tracing/utils.py

@@ -1,4 +1,5 @@
 import os
+from typing import Dict, Any
 from datetime import datetime, timezone
 from enum import Enum
 from time import perf_counter
@@ -183,3 +184,10 @@ def replace_self_with_class_name(obj):
         return f"<{obj.__class__.__name__}>"
     except:
         return f"<self>"
+
+
+def prepare_tool_call_input_parameters(output: Any) -> Dict[str, Any]:
+    res = make_json_serializable(output)
+    if res and not isinstance(res, dict):
+        res = {"output": res}
+    return res

deepeval/utils.py

@@ -10,6 +10,7 @@ import asyncio
 import nest_asyncio
 import uuid
 import math
+import logging
 
 from contextvars import ContextVar
 from enum import Enum
@@ -747,3 +748,23 @@ my_theme = Theme(
     }
 )
 custom_console = Console(theme=my_theme)
+
+
+def format_error_text(
+    exc: BaseException, *, with_stack: bool | None = None
+) -> str:
+    if with_stack is None:
+        with_stack = logging.getLogger("deepeval").isEnabledFor(logging.DEBUG)
+
+    text = f"{type(exc).__name__}: {exc}"
+
+    if with_stack:
+        import traceback
+
+        text += "\n" + "".join(
+            traceback.format_exception(type(exc), exc, exc.__traceback__)
+        )
+    elif get_settings().DEEPEVAL_VERBOSE_MODE:
+        text += " (Run with LOG_LEVEL=DEBUG for stack trace.)"
+
+    return text

{deepeval-3.6.6.dist-info → deepeval-3.6.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.6
+Version: 3.6.8
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0