deepeval 3.5.9__tar.gz → 3.6.0__tar.gz

{deepeval-3.5.9 → deepeval-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.9
+Version: 3.6.0
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.6.0/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.6.0"

{deepeval-3.5.9 → deepeval-3.6.0}/deepeval/config/settings_manager.py

@@ -15,7 +15,7 @@ from enum import Enum
 from pydantic import SecretStr
 from deepeval.config.settings import get_settings, _SAVE_RE
 from deepeval.cli.dotenv_handler import DotenvHandler
-from deepeval.utils import bool_to_env_str
+from deepeval.config.utils import bool_to_env_str
 
 logger = logging.getLogger(__name__)
 StrOrEnum = Union[str, Enum]

deepeval-3.6.0/deepeval/contextvars.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Optional
+
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden
+
+
+CURRENT_GOLDEN: ContextVar[Optional[Golden]] = ContextVar(
+    "CURRENT_GOLDEN", default=None
+)
+
+
+def set_current_golden(golden: Optional[Golden]):
+    return CURRENT_GOLDEN.set(golden)
+
+
+def get_current_golden() -> Optional[Golden]:
+    return CURRENT_GOLDEN.get()
+
+
+def reset_current_golden(token) -> None:
+    CURRENT_GOLDEN.reset(token)

deepeval-3.6.0/deepeval/dataset/__init__.py

@@ -0,0 +1,11 @@
+from deepeval.contextvars import get_current_golden
+from .dataset import EvaluationDataset
+from .golden import Golden, ConversationalGolden
+
+
+__all__ = [
+    "EvaluationDataset",
+    "Golden",
+    "ConversationalGolden",
+    "get_current_golden",
+]

{deepeval-3.5.9 → deepeval-3.6.0}/deepeval/evaluate/execute.py

@@ -42,6 +42,7 @@ from deepeval.tracing.api import (
     BaseApiSpan,
 )
 from deepeval.dataset import Golden
+from deepeval.contextvars import set_current_golden, reset_current_golden
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.utils import copy_metrics
 from deepeval.utils import (
@@ -1480,6 +1481,7 @@ def execute_agentic_test_cases_from_loop(
         )
 
         for golden in goldens:
+            token = set_current_golden(golden)
             with capture_evaluation_run("golden"):
                 # yield golden
                 count += 1
@@ -1492,8 +1494,14 @@ def execute_agentic_test_cases_from_loop(
                     _progress=progress,
                     _pbar_callback_id=pbar_tags_id,
                 ):
-                    yield golden
-                    current_trace: Trace = current_trace_context.get()
+                    try:
+                        # yield golden to user code
+                        yield golden
+                        # control has returned from user code without error, capture trace now
+                        current_trace: Trace = current_trace_context.get()
+                    finally:
+                        # after user code returns control, always reset the context
+                        reset_current_golden(token)
 
                 update_pbar(progress, pbar_tags_id)
                 update_pbar(progress, pbar_id)
@@ -1849,6 +1857,7 @@ def a_execute_agentic_test_cases_from_loop(
 
         try:
             for index, golden in enumerate(goldens):
+                token = set_current_golden(golden)
                 current_golden_ctx.update(
                     {
                         "index": index,
@@ -1857,7 +1866,10 @@ def a_execute_agentic_test_cases_from_loop(
                     }
                 )
                 prev_task_length = len(created_tasks)
-                yield golden
+                try:
+                    yield golden
+                finally:
+                    reset_current_golden(token)
                 # if this golden created no tasks, bump bars now
                 if len(created_tasks) == prev_task_length:
                     update_pbar(progress, pbar_callback_id)

{deepeval-3.5.9 → deepeval-3.6.0}/deepeval/openai_agents/__init__.py

@@ -1,6 +1,7 @@
 from deepeval.openai_agents.callback_handler import DeepEvalTracingProcessor
-from deepeval.openai_agents.runner import Runner
-from deepeval.openai_agents.patch import function_tool
 from deepeval.openai_agents.agent import DeepEvalAgent as Agent
+from deepeval.openai_agents.patch import function_tool
+
+# from deepeval.openai_agents.runner import Runner
 
-__all__ = ["DeepEvalTracingProcessor", "Runner", "function_tool", "Agent"]
+__all__ = ["DeepEvalTracingProcessor", "Agent", "function_tool"]

deepeval-3.6.0/deepeval/openai_agents/agent.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Generic, TypeVar, List
+
+from deepeval.prompt import Prompt
+from deepeval.metrics import BaseMetric
+from deepeval.tracing.types import LlmSpan
+
+try:
+    from agents.agent import Agent as BaseAgent
+    from deepeval.openai_agents.patch import (
+        patch_default_agent_runner_get_model,
+    )
+except Exception as e:
+    raise RuntimeError(
+        "openai-agents is required for this integration. Please install it."
+    ) from e
+
+TContext = TypeVar("TContext")
+
+
+@dataclass
+class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
+    """
+    A subclass of agents.Agent.
+    """
+
+    llm_metric_collection: str = None
+    llm_metrics: List[BaseMetric] = None
+    confident_prompt: Prompt = None
+    agent_metrics: List[BaseMetric] = None
+    agent_metric_collection: str = None
+
+    def __post_init__(self):
+        patch_default_agent_runner_get_model()

deepeval-3.6.0/deepeval/openai_agents/callback_handler.py

@@ -0,0 +1,135 @@
+from deepeval.tracing.tracing import (
+    Observer,
+    current_span_context,
+    trace_manager,
+)
+from deepeval.openai_agents.extractors import *
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.utils import make_json_serializable
+from time import perf_counter
+from deepeval.tracing.types import TraceSpanStatus
+
+try:
+    from agents.tracing import Span, Trace, TracingProcessor
+    from agents.tracing.span_data import (
+        AgentSpanData,
+        CustomSpanData,
+        FunctionSpanData,
+        GenerationSpanData,
+        GuardrailSpanData,
+        HandoffSpanData,
+        ResponseSpanData,
+        SpanData,
+    )
+    from deepeval.openai_agents.patch import (
+        patch_default_agent_run_single_turn,
+        patch_default_agent_run_single_turn_streamed,
+    )
+
+    openai_agents_available = True
+except ImportError:
+    openai_agents_available = False
+
+
+def _check_openai_agents_available():
+    if not openai_agents_available:
+        raise ImportError(
+            "openai-agents is required for this integration. Install it via your package manager"
+        )
+
+
+class DeepEvalTracingProcessor(TracingProcessor):
+    def __init__(self) -> None:
+        _check_openai_agents_available()
+        patch_default_agent_run_single_turn()
+        patch_default_agent_run_single_turn_streamed()
+        self.span_observers: dict[str, Observer] = {}
+
+    def on_trace_start(self, trace: "Trace") -> None:
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _thread_id = trace_dict.get("group_id")
+        _trace_name = trace_dict.get("workflow_name")
+        _trace_metadata = trace_dict.get("metadata")
+
+        _trace = trace_manager.start_new_trace(trace_uuid=str(_trace_uuid))
+        _trace.thread_id = str(_thread_id)
+        _trace.name = str(_trace_name)
+        _trace.metadata = make_json_serializable(_trace_metadata)
+        current_trace_context.set(_trace)
+
+        trace_manager.add_span(  # adds a dummy root span
+            BaseSpan(
+                uuid=_trace_uuid,
+                trace_uuid=_trace_uuid,
+                parent_uuid=None,
+                start_time=perf_counter(),
+                name=_trace_name,
+                status=TraceSpanStatus.IN_PROGRESS,
+                children=[],
+            )
+        )
+
+    def on_trace_end(self, trace: "Trace") -> None:
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _trace_name = trace_dict.get("workflow_name")
+
+        trace_manager.remove_span(_trace_uuid)  # removing the dummy root span
+        trace_manager.end_trace(_trace_uuid)
+        current_trace_context.set(None)
+
+    def on_span_start(self, span: "Span") -> None:
+        if not span.started_at:
+            return
+        current_span = current_span_context.get()
+        if current_span and isinstance(current_span, LlmSpan):
+            return
+
+        span_type = self.get_span_kind(span.span_data)
+        observer = Observer(span_type=span_type, func_name="NA")
+        if span_type == "llm":
+            observer.observe_kwargs["model"] = "temporary model"
+        observer.update_span_properties = (
+            lambda span_type: update_span_properties(span_type, span.span_data)
+        )
+        self.span_observers[span.span_id] = observer
+        observer.__enter__()
+
+    def on_span_end(self, span: "Span") -> None:
+        update_trace_properties_from_span_data(
+            current_trace_context.get(), span.span_data
+        )
+
+        current_span = current_span_context.get()
+        if current_span and isinstance(current_span, LlmSpan):
+            update_span_properties(current_span, span.span_data)
+            return
+        observer = self.span_observers.pop(span.span_id, None)
+        if observer:
+            observer.__exit__(None, None, None)
+
+    def force_flush(self) -> None:
+        pass
+
+    def shutdown(self) -> None:
+        pass
+
+    def get_span_kind(self, span_data: "SpanData") -> str:
+        if isinstance(span_data, AgentSpanData):
+            return "agent"
+        if isinstance(span_data, FunctionSpanData):
+            return "tool"
+        if isinstance(span_data, MCPListToolsSpanData):
+            return "tool"
+        if isinstance(span_data, GenerationSpanData):
+            return "llm"
+        if isinstance(span_data, ResponseSpanData):
+            return "llm"
+        if isinstance(span_data, HandoffSpanData):
+            return "custom"
+        if isinstance(span_data, CustomSpanData):
+            return "base"
+        if isinstance(span_data, GuardrailSpanData):
+            return "base"
+        return "base"

{deepeval-3.5.9 → deepeval-3.6.0}/deepeval/openai_agents/extractors.py

@@ -1,9 +1,10 @@
+from deepeval.tracing.types import Trace
 from openai.types.responses.response_input_item_param import (
     FunctionCallOutput,
     Message,
 )
 from openai.types.responses.response_output_message_param import Content
-from typing import Union, List
+from typing import Union, List, Optional
 from openai.types.responses import (
     ResponseFunctionToolCallParam,
     ResponseOutputMessageParam,
@@ -25,6 +26,8 @@ from deepeval.tracing.types import (
 )
 import json
 
+from deepeval.tracing.utils import make_json_serializable
+
 try:
     from agents import MCPListToolsSpanData
     from agents.tracing.span_data import (
@@ -89,13 +92,17 @@ def update_span_properties_from_response_span_data(
         return
     # Extract usage tokens
     usage = response.usage
+    cached_input_tokens = None
+    ouptut_reasoning_tokens = None
     if usage:
         output_tokens = usage.output_tokens
         input_tokens = usage.input_tokens
         cached_input_tokens = usage.input_tokens_details.cached_tokens
         ouptut_reasoning_tokens = usage.output_tokens_details.reasoning_tokens
     # Get input and output
-    input = parse_response_input(span_data.input)
+    input = parse_response_input(
+        span_data.input, span_data.response.instructions
+    )
     raw_output = parse_response_output(response.output)
     output = (
         raw_output if isinstance(raw_output, str) else json.dumps(raw_output)
@@ -112,6 +119,23 @@ def update_span_properties_from_response_span_data(
     span.input = input
     span.output = output
     span.name = "LLM Generation"
+    response_dict = response.model_dump(exclude_none=True, mode="json")
+    span.metadata["invocation_params"] = {
+        k: v
+        for k, v in response_dict.items()
+        if k
+        in (
+            "max_output_tokens",
+            "parallel_tool_calls",
+            "reasoning",
+            "temperature",
+            "text",
+            "tool_choice",
+            "tools",
+            "top_p",
+            "truncation",
+        )
+    }
 
 
 def update_span_properties_from_generation_span_data(
@@ -136,6 +160,11 @@ def update_span_properties_from_generation_span_data(
     span.input = input
     span.output = output
     span.name = "LLM Generation"
+    span.metadata["invocation_params"] = {
+        "model_config": make_json_serializable(
+            generation_span_data.model_config
+        ),
+    }
 
 
 ########################################################
@@ -191,8 +220,6 @@ def update_span_properties_from_agent_span_data(
     if agent_span_data.output_type:
         metadata["output_type"] = agent_span_data.output_type
     span.metadata = metadata
-    span.input = None
-    span.output = None
 
 
 ########################################################
@@ -238,10 +265,30 @@ def update_span_properties_from_guardrail_span_data(
 ########################################################
 
 
-def parse_response_input(input: Union[str, List[ResponseInputItemParam]]):
-    if isinstance(input, str):
-        return input
+def parse_response_input(
+    input: Union[str, List[ResponseInputItemParam]],
+    instructions: Optional[Union[str, List[ResponseInputItemParam]]] = None,
+):
+
     processed_input = []
+
+    if isinstance(input, str) and isinstance(instructions, str):
+        return [
+            {"type": "message", "role": "system", "content": instructions},
+            {"type": "message", "role": "user", "content": input},
+        ]
+    elif isinstance(input, list) and isinstance(instructions, list):
+        input = instructions + input
+    elif isinstance(input, list) and isinstance(instructions, str):
+        processed_input += [
+            {"type": "message", "role": "system", "content": instructions}
+        ]
+    elif isinstance(input, str) and isinstance(instructions, list):
+        processed_input += [
+            {"type": "message", "role": "user", "content": input}
+        ]
+        input = instructions
+
     for item in input:
         if "type" not in item:
             if "role" in item and "content" in item:
@@ -365,3 +412,32 @@ def parse_function_call(
         "name": function_call.name,
         "arguments": function_call.arguments,
     }
+
+
+def update_trace_properties_from_span_data(
+    trace: Trace,
+    span_data: Union["ResponseSpanData", "GenerationSpanData"],
+):
+    if isinstance(span_data, ResponseSpanData):
+        if not trace.input:
+            trace.input = parse_response_input(
+                span_data.input, span_data.response.instructions
+            )
+        raw_output = parse_response_output(span_data.response.output)
+        output = (
+            raw_output
+            if isinstance(raw_output, str)
+            else json.dumps(raw_output)
+        )
+        trace.output = output
+
+    elif isinstance(span_data, GenerationSpanData):
+        if not trace.input:
+            trace.input = span_data.input
+        raw_output = span_data.output
+        output = (
+            raw_output
+            if isinstance(raw_output, str)
+            else json.dumps(raw_output)
+        )
+        trace.output = output