deepeval 3.5.4__py3-none-any.whl → 3.5.6__py3-none-any.whl

deepeval/openai_agents/callback_handler.py

@@ -1,9 +1,13 @@
 from deepeval.tracing.tracing import (
     Observer,
     current_span_context,
+    trace_manager,
 )
 from deepeval.openai_agents.extractors import *
 from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.utils import make_json_serializable
+from time import perf_counter
+from deepeval.tracing.types import TraceSpanStatus
 
 try:
     from agents.tracing import Span, Trace, TracingProcessor
@@ -33,30 +37,57 @@ def _check_openai_agents_available():
 class DeepEvalTracingProcessor(TracingProcessor):
     def __init__(self) -> None:
         _check_openai_agents_available()
-        self.root_span_observers: dict[str, Observer] = {}
         self.span_observers: dict[str, Observer] = {}
 
     def on_trace_start(self, trace: "Trace") -> None:
-        pass
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _thread_id = trace_dict.get("group_id")
+        _trace_name = trace_dict.get("workflow_name")
+        _trace_metadata = trace_dict.get("metadata")
+
+        if _thread_id or _trace_metadata:
+            _trace = trace_manager.start_new_trace(trace_uuid=str(_trace_uuid))
+            _trace.thread_id = str(_thread_id)
+            _trace.name = str(_trace_name)
+            _trace.metadata = make_json_serializable(_trace_metadata)
+            current_trace_context.set(_trace)
+
+            trace_manager.add_span(  # adds a dummy root span
+                BaseSpan(
+                    uuid=_trace_uuid,
+                    trace_uuid=_trace_uuid,
+                    parent_uuid=None,
+                    start_time=perf_counter(),
+                    name=_trace_name,
+                    status=TraceSpanStatus.IN_PROGRESS,
+                    children=[],
+                )
+            )
+        else:
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.name = str(_trace_name)
 
     def on_trace_end(self, trace: "Trace") -> None:
-        pass
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _thread_id = trace_dict.get("group_id")
+        _trace_name = trace_dict.get("workflow_name")
+        _trace_metadata = trace_dict.get("metadata")
+
+        if _thread_id or _trace_metadata:
+            trace_manager.remove_span(
+                _trace_uuid
+            )  # removing the dummy root span
+            trace_manager.end_trace(_trace_uuid)
+            current_trace_context.set(None)
 
     def on_span_start(self, span: "Span") -> None:
         if not span.started_at:
             return
         span_type = self.get_span_kind(span.span_data)
-        if span_type == "agent":
-            if isinstance(span.span_data, AgentSpanData):
-                current_trace = current_trace_context.get()
-                if current_trace:
-                    current_trace.name = span.span_data.name
-
-        if span_type == "tool":
-            return
-        elif span_type == "llm":
-            return
-        else:
+        if span_type and span_type == "agent":
             observer = Observer(span_type=span_type, func_name="NA")
             observer.update_span_properties = (
                 lambda base_span: update_span_properties(
@@ -68,13 +99,13 @@ class DeepEvalTracingProcessor(TracingProcessor):
 
     def on_span_end(self, span: "Span") -> None:
         span_type = self.get_span_kind(span.span_data)
-        if span_type == "llm":
+        if span_type and span_type == "agent":
             current_span = current_span_context.get()
             if current_span:
                 update_span_properties(current_span, span.span_data)
-        observer = self.span_observers.pop(span.span_id, None)
-        if observer:
-            observer.__exit__(None, None, None)
+            observer = self.span_observers.pop(span.span_id, None)
+            if observer:
+                observer.__exit__(None, None, None)
 
     def force_flush(self) -> None:
         pass
@@ -85,18 +116,19 @@ class DeepEvalTracingProcessor(TracingProcessor):
     def get_span_kind(self, span_data: "SpanData") -> str:
         if isinstance(span_data, AgentSpanData):
             return "agent"
-        if isinstance(span_data, FunctionSpanData):
-            return "tool"
-        if isinstance(span_data, MCPListToolsSpanData):
-            return "tool"
-        if isinstance(span_data, GenerationSpanData):
-            return "llm"
-        if isinstance(span_data, ResponseSpanData):
-            return "llm"
-        if isinstance(span_data, HandoffSpanData):
-            return "custom"
-        if isinstance(span_data, CustomSpanData):
-            return "base"
-        if isinstance(span_data, GuardrailSpanData):
-            return "base"
-        return "base"
+        # if isinstance(span_data, FunctionSpanData):
+        #     return "tool"
+        # if isinstance(span_data, MCPListToolsSpanData):
+        #     return "tool"
+        # if isinstance(span_data, GenerationSpanData):
+        #     return "llm"
+        # if isinstance(span_data, ResponseSpanData):
+        #     return "llm"
+        # if isinstance(span_data, HandoffSpanData):
+        #     return "custom"
+        # if isinstance(span_data, CustomSpanData):
+        #     return "base"
+        # if isinstance(span_data, GuardrailSpanData):
+        #     return "base"
+        # return "base"
+        return None

deepeval/openai_agents/runner.py

@@ -1,114 +1,335 @@
 from __future__ import annotations
 
 from dataclasses import replace
+from typing import List, Any, Union, Optional
+
+try:
+    from agents import (
+        RunConfig,
+        RunResult,
+        RunResultStreaming,
+        Runner as AgentsRunner,
+    )
+    from agents.agent import Agent
+    from agents.models.interface import ModelProvider
+    from agents.items import TResponseInputItem
+    from agents.lifecycle import RunHooks
+    from agents.memory import Session
+    from agents.run import DEFAULT_MAX_TURNS
+    from agents.run import AgentRunner
+    from agents.run_context import TContext
+    from agents.models.interface import Model
+
+    agents_available = True
+except:
+    agents_available = False
+
+
+def is_agents_available():
+    if not agents_available:
+        raise ImportError(
+            "agents is required for this integration. Install it via your package manager"
+        )
+
 
-from agents import (
-    Runner as BaseRunner,
-    RunConfig,
-    RunResult,
-    RunResultStreaming,
-)
 from deepeval.tracing.tracing import Observer
 from deepeval.tracing.context import current_span_context, current_trace_context
 
 # Import observed provider/model helpers from our agent module
-from deepeval.openai_agents.agent import _ObservedProvider
+from deepeval.metrics import BaseMetric
+from deepeval.openai_agents.agent import _ObservedModel
 
+_PATCHED_DEFAULT_GET_MODEL = False
 
-class Runner(BaseRunner):
-    """
-    Extends Runner to:
-      - capture metric_collection/metrics at run entry for tracing
-      - ensure RunConfig.model_provider is wrapped to return observed Models
-        so string-based model lookups are also instrumented.
-    """
+
+def _patch_default_agent_runner_get_model():
+    global _PATCHED_DEFAULT_GET_MODEL
+    if _PATCHED_DEFAULT_GET_MODEL:
+        return
+
+    original_get_model = AgentRunner._get_model
 
     @classmethod
-    async def run(cls, *args, **kwargs) -> RunResult:
-        metric_collection = kwargs.pop("metric_collection", None)
-        metrics = kwargs.pop("metrics", None)
+    def patched_get_model(
+        cls, agent: Agent[Any], run_config: RunConfig
+    ) -> Model:
+        model = original_get_model(agent, run_config)
 
-        # Ensure the model provider is wrapped so _get_model(...) uses observed Models
-        starting_agent = (
-            args[0] if len(args) > 0 else kwargs.get("starting_agent")
+        # Extract attributes from agent if it's a DeepEvalAgent
+        llm_metrics = getattr(agent, "llm_metrics", None)
+        llm_metric_collection = getattr(agent, "llm_metric_collection", None)
+        confident_prompt = getattr(agent, "confident_prompt", None)
+        model = _ObservedModel(
+            inner=model,
+            llm_metric_collection=llm_metric_collection,
+            llm_metrics=llm_metrics,
+            confident_prompt=confident_prompt,
         )
-        run_config: RunConfig | None = kwargs.get("run_config")
-        if run_config is None:
-            run_config = RunConfig()
-            kwargs["run_config"] = run_config
-
-        if run_config.model_provider is not None:
-            run_config.model_provider = _ObservedProvider(
-                run_config.model_provider,
-                metrics=getattr(starting_agent, "metrics", None) or metrics,
-                metric_collection=getattr(
-                    starting_agent, "metric_collection", None
-                )
-                or metric_collection,
-                deepeval_prompt=getattr(
-                    starting_agent, "deepeval_prompt", None
-                ),
-            )
 
-        input_val = args[1] if len(args) >= 2 else kwargs.get("input", None)
+        return model
+
+    # Replace the method
+    AgentRunner._get_model = patched_get_model
+    _PATCHED_DEFAULT_GET_MODEL = True
+
+
+if agents_available:
+    _patch_default_agent_runner_get_model()
+
+
+class Runner(AgentsRunner):
+
+    @classmethod
+    async def run(
+        cls,
+        starting_agent: Agent[TContext],
+        input: Union[str, list[TResponseInputItem]],
+        *,
+        context: Optional[TContext] = None,
+        max_turns: int = DEFAULT_MAX_TURNS,
+        hooks: Optional[RunHooks[TContext]] = None,
+        run_config: Optional[RunConfig] = None,
+        previous_response_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        session: Optional[Session] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs,  # backwards compatibility
+    ) -> RunResult:
+        is_agents_available()
+        # _patch_default_agent_runner_get_model()
+
         with Observer(
             span_type="custom",
             metric_collection=metric_collection,
             metrics=metrics,
             func_name="run",
-            function_kwargs={"input": input_val},
+            function_kwargs={"input": input},  # also set below
         ) as observer:
+            update_trace_attributes(
+                name=name,
+                tags=tags,
+                metadata=metadata,
+                thread_id=thread_id,
+                user_id=user_id,
+                metric_collection=metric_collection,
+                metrics=metrics,
+            )
             current_span = current_span_context.get()
             current_trace = current_trace_context.get()
-            current_trace.input = input_val
+            if not current_trace.input:
+                current_trace.input = input
             if current_span:
-                current_span.input = input_val
-            res = await super().run(*args, **kwargs)
-            current_trace.output = str(res)
-            observer.result = str(res)
+                current_span.input = input
+            res = await super().run(
+                starting_agent,
+                input,
+                context=context,
+                max_turns=max_turns,
+                hooks=hooks,
+                run_config=run_config,
+                previous_response_id=previous_response_id,
+                conversation_id=conversation_id,
+                session=session,
+                **kwargs,  # backwards compatibility
+            )
+            current_trace_thread_id = current_trace_context.get().thread_id
+            _output = None
+            if current_trace_thread_id:
+                _output = res.final_output
+            else:
+                _output = str(res)
+            observer.result = _output
+            update_trace_attributes(output=_output)
         return res
 
     @classmethod
-    def run_sync(cls, *args, **kwargs) -> RunResult:
-        metric_collection = kwargs.pop("metric_collection", None)
-        metrics = kwargs.pop("metrics", None)
+    def run_sync(
+        cls,
+        starting_agent: Agent[TContext],
+        input: Union[str, list[TResponseInputItem]],
+        *,
+        context: Optional[TContext] = None,
+        max_turns: int = DEFAULT_MAX_TURNS,
+        hooks: Optional[RunHooks[TContext]] = None,
+        run_config: Optional[RunConfig] = None,
+        previous_response_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        session: Optional[Session] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs,
+    ) -> RunResult:
+        is_agents_available()
 
-        starting_agent = (
-            args[0] if len(args) > 0 else kwargs.get("starting_agent")
-        )
-        run_config: RunConfig | None = kwargs.get("run_config")
-        if run_config is None:
-            run_config = RunConfig()
-            kwargs["run_config"] = run_config
-
-        if run_config.model_provider is not None:
-            run_config.model_provider = _ObservedProvider(
-                run_config.model_provider,
-                metrics=getattr(starting_agent, "metrics", None) or metrics,
-                metric_collection=getattr(
-                    starting_agent, "metric_collection", None
-                )
-                or metric_collection,
-                deepeval_prompt=getattr(
-                    starting_agent, "deepeval_prompt", None
-                ),
-            )
-
-        input_val = args[1] if len(args) >= 2 else kwargs.get("input", None)
         with Observer(
             span_type="custom",
             metric_collection=metric_collection,
             metrics=metrics,
             func_name="run_sync",
-            function_kwargs={"input": input_val},
+            function_kwargs={"input": input},  # also set below
         ) as observer:
+            update_trace_attributes(
+                name=name,
+                tags=tags,
+                metadata=metadata,
+                thread_id=thread_id,
+                user_id=user_id,
+                metric_collection=metric_collection,
+                metrics=metrics,
+            )
+
             current_span = current_span_context.get()
             current_trace = current_trace_context.get()
-            current_trace.input = input_val
+            if not current_trace.input:
+                current_trace.input = input
             if current_span:
-                current_span.input = input_val
-            res = super().run_sync(*args, **kwargs)
-            current_trace.output = str(res)
-            observer.result = str(res)
+                current_span.input = input
+            res = super().run_sync(
+                starting_agent,
+                input,
+                context=context,
+                max_turns=max_turns,
+                hooks=hooks,
+                run_config=run_config,
+                previous_response_id=previous_response_id,
+                conversation_id=conversation_id,
+                session=session,
+                **kwargs,  # backwards compatibility
+            )
+            current_trace_thread_id = current_trace_context.get().thread_id
+            _output = None
+            if current_trace_thread_id:
+                _output = res.final_output
+            else:
+                _output = str(res)
+            update_trace_attributes(output=_output)
+            observer.result = _output
 
         return res
+
+    @classmethod
+    def run_streamed(
+        cls,
+        starting_agent: Agent[TContext],
+        input: Union[str, list[TResponseInputItem]],
+        *,
+        context: Optional[TContext] = None,
+        max_turns: int = DEFAULT_MAX_TURNS,
+        hooks: Optional[RunHooks[TContext]] = None,
+        run_config: Optional[RunConfig] = None,
+        previous_response_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        session: Optional[Session] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs,  # backwards compatibility
+    ) -> RunResultStreaming:
+        is_agents_available()
+        # Manually enter observer; we'll exit when streaming finishes
+        observer = Observer(
+            span_type="custom",
+            metric_collection=metric_collection,
+            metrics=metrics,
+            func_name="run_streamed",
+            function_kwargs={"input": input},
+        )
+        observer.__enter__()
+
+        update_trace_attributes(
+            name=name,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            user_id=user_id,
+            metric_collection=metric_collection,
+            metrics=metrics,
+        )
+        current_trace = current_trace_context.get()
+        if not current_trace.input:
+            current_trace.input = input
+
+        current_span = current_span_context.get()
+        if current_span:
+            current_span.input = input
+
+        res = super().run_streamed(
+            starting_agent,
+            input,
+            context=context,
+            max_turns=max_turns,
+            hooks=hooks,
+            run_config=run_config,
+            previous_response_id=previous_response_id,
+            conversation_id=conversation_id,
+            session=session,
+            **kwargs,  # backwards compatibility
+        )
+
+        # Runtime-patch stream_events so the observer closes only after streaming completes
+        orig_stream_events = res.stream_events
+
+        async def _patched_stream_events(self: RunResultStreaming):
+            try:
+                async for event in orig_stream_events():
+                    yield event
+                observer.result = self.final_output
+                update_trace_attributes(output=self.final_output)
+            except Exception as e:
+                observer.__exit__(type(e), e, e.__traceback__)
+                raise
+            finally:
+                observer.__exit__(None, None, None)
+
+        from types import MethodType as _MethodType
+
+        res.stream_events = _MethodType(_patched_stream_events, res)
+
+        return res
+
+
+def update_trace_attributes(
+    input: Any = None,
+    output: Any = None,
+    name: str = None,
+    tags: List[str] = None,
+    metadata: dict = None,
+    thread_id: str = None,
+    user_id: str = None,
+    metric_collection: str = None,
+    metrics: List[BaseMetric] = None,
+):
+    current_trace = current_trace_context.get()
+    if input:
+        current_trace.input = input
+    if output:
+        current_trace.output = output
+    if name:
+        current_trace.name = name
+    if tags:
+        current_trace.tags = tags
+    if metadata:
+        current_trace.metadata = metadata
+    if thread_id:
+        current_trace.thread_id = thread_id
+    if user_id:
+        current_trace.user_id = user_id
+    if metric_collection:
+        current_trace.metric_collection = metric_collection
+    if metrics:
+        current_trace.metrics = metrics

deepeval/scorer/scorer.py

@@ -223,7 +223,7 @@ class Scorer:
         Right now we are using score_one method under the hood. Instead of scoring multiple predictions for faithfulness.
         """
         try:
-            from deepeval.models import SummaCModels
+            from deepeval.models.summac_model import SummaCModels
         except Exception as e:
             print(f"SummaCZS model can not be loaded.\n{e}")
 
@@ -326,7 +326,7 @@ class Scorer:
         from sentence_transformers import util
 
         try:
-            from deepeval.models import (
+            from deepeval.models.answer_relevancy_model import (
                 AnswerRelevancyModel,
                 CrossEncoderAnswerRelevancyModel,
             )

deepeval/tracing/tracing.py

@@ -8,9 +8,6 @@ import random
 import atexit
 import queue
 import uuid
-import os
-import json
-import time
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
@@ -496,6 +493,7 @@ class TraceManager:
                     asyncio.gather(*pending, return_exceptions=True)
                 )
             self.flush_traces(remaining_trace_request_bodies)
+            loop.run_until_complete(loop.shutdown_asyncgens())
             loop.close()
 
     def flush_traces(

{deepeval-3.5.4.dist-info → deepeval-3.5.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.4
+Version: 3.5.6
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -186,6 +186,8 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 
 ## Installation
 
+Deepeval works with **Python>=3.9+**.
+
 ```
 pip install -U deepeval
 ```