deepeval 3.5.4__py3-none-any.whl → 3.5.5__py3-none-any.whl

deepeval/openai_agents/runner.py

@@ -1,100 +1,188 @@
 from __future__ import annotations
 
 from dataclasses import replace
+from typing import List, Any, Union, Optional
+
+try:
+    from agents import (
+        RunConfig,
+        RunResult,
+        RunResultStreaming,
+        Runner as AgentsRunner,
+    )
+    from agents.agent import Agent
+    from agents.models.interface import ModelProvider
+    from agents.items import TResponseInputItem
+    from agents.lifecycle import RunHooks
+    from agents.memory import Session
+    from agents.run import DEFAULT_MAX_TURNS
+    from agents.run import AgentRunner
+    from agents.run_context import TContext
+    from agents.models.interface import Model
+
+    agents_available = True
+except:
+    agents_available = False
+
+
+def is_agents_available():
+    if not agents_available:
+        raise ImportError(
+            "agents is required for this integration. Install it via your package manager"
+        )
+
 
-from agents import (
-    Runner as BaseRunner,
-    RunConfig,
-    RunResult,
-    RunResultStreaming,
-)
 from deepeval.tracing.tracing import Observer
 from deepeval.tracing.context import current_span_context, current_trace_context
 
 # Import observed provider/model helpers from our agent module
-from deepeval.openai_agents.agent import _ObservedProvider
+from deepeval.metrics import BaseMetric
+from deepeval.openai_agents.agent import _ObservedModel
 
+_PATCHED_DEFAULT_GET_MODEL = False
 
-class Runner(BaseRunner):
-    """
-    Extends Runner to:
-      - capture metric_collection/metrics at run entry for tracing
-      - ensure RunConfig.model_provider is wrapped to return observed Models
-        so string-based model lookups are also instrumented.
-    """
+
+def _patch_default_agent_runner_get_model():
+    global _PATCHED_DEFAULT_GET_MODEL
+    if _PATCHED_DEFAULT_GET_MODEL:
+        return
+
+    original_get_model = AgentRunner._get_model
 
     @classmethod
-    async def run(cls, *args, **kwargs) -> RunResult:
-        metric_collection = kwargs.pop("metric_collection", None)
-        metrics = kwargs.pop("metrics", None)
+    def patched_get_model(
+        cls, agent: Agent[Any], run_config: RunConfig
+    ) -> Model:
+        model = original_get_model(agent, run_config)
 
-        # Ensure the model provider is wrapped so _get_model(...) uses observed Models
-        starting_agent = (
-            args[0] if len(args) > 0 else kwargs.get("starting_agent")
+        # Extract attributes from agent if it's a DeepEvalAgent
+        llm_metrics = getattr(agent, "llm_metrics", None)
+        llm_metric_collection = getattr(agent, "llm_metric_collection", None)
+        confident_prompt = getattr(agent, "confident_prompt", None)
+        model = _ObservedModel(
+            inner=model,
+            llm_metric_collection=llm_metric_collection,
+            llm_metrics=llm_metrics,
+            confident_prompt=confident_prompt,
         )
-        run_config: RunConfig | None = kwargs.get("run_config")
-        if run_config is None:
-            run_config = RunConfig()
-            kwargs["run_config"] = run_config
-
-        if run_config.model_provider is not None:
-            run_config.model_provider = _ObservedProvider(
-                run_config.model_provider,
-                metrics=getattr(starting_agent, "metrics", None) or metrics,
-                metric_collection=getattr(
-                    starting_agent, "metric_collection", None
-                )
-                or metric_collection,
-                deepeval_prompt=getattr(
-                    starting_agent, "deepeval_prompt", None
-                ),
-            )
 
-        input_val = args[1] if len(args) >= 2 else kwargs.get("input", None)
+        return model
+
+    # Replace the method
+    AgentRunner._get_model = patched_get_model
+    _PATCHED_DEFAULT_GET_MODEL = True
+
+
+if agents_available:
+    _patch_default_agent_runner_get_model()
+
+
+class Runner(AgentsRunner):
+
+    @classmethod
+    async def run(
+        cls,
+        starting_agent: Agent[TContext],
+        input: Union[str, list[TResponseInputItem]],
+        *,
+        context: Optional[TContext] = None,
+        max_turns: int = DEFAULT_MAX_TURNS,
+        hooks: Optional[RunHooks[TContext]] = None,
+        run_config: Optional[RunConfig] = None,
+        previous_response_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        session: Optional[Session] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs,  # backwards compatibility
+    ) -> RunResult:
+        is_agents_available()
+        # _patch_default_agent_runner_get_model()
+
         with Observer(
             span_type="custom",
             metric_collection=metric_collection,
             metrics=metrics,
             func_name="run",
-            function_kwargs={"input": input_val},
+            function_kwargs={"input": input},
         ) as observer:
+            update_trace_attributes(
+                input=input,
+                name=name,
+                tags=tags,
+                metadata=metadata,
+                thread_id=thread_id,
+                user_id=user_id,
+                metric_collection=metric_collection,
+                metrics=metrics,
+            )
             current_span = current_span_context.get()
             current_trace = current_trace_context.get()
-            current_trace.input = input_val
+            current_trace.input = input
             if current_span:
-                current_span.input = input_val
-            res = await super().run(*args, **kwargs)
-            current_trace.output = str(res)
-            observer.result = str(res)
+                current_span.input = input
+            res = await super().run(
+                starting_agent,
+                input,
+                context=context,
+                max_turns=max_turns,
+                hooks=hooks,
+                run_config=run_config,
+                previous_response_id=previous_response_id,
+                conversation_id=conversation_id,
+                session=session,
+                **kwargs,  # backwards compatibility
+            )
+            _output = None
+            if thread_id:
+                _output = res.final_output
+            else:
+                _output = str(res)
+            observer.result = _output
+            update_trace_attributes(output=_output)
         return res
 
     @classmethod
-    def run_sync(cls, *args, **kwargs) -> RunResult:
-        metric_collection = kwargs.pop("metric_collection", None)
-        metrics = kwargs.pop("metrics", None)
+    def run_sync(
+        cls,
+        starting_agent: Agent[TContext],
+        input: Union[str, list[TResponseInputItem]],
+        *,
+        context: Optional[TContext] = None,
+        max_turns: int = DEFAULT_MAX_TURNS,
+        hooks: Optional[RunHooks[TContext]] = None,
+        run_config: Optional[RunConfig] = None,
+        previous_response_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        session: Optional[Session] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs,
+    ) -> RunResult:
+        is_agents_available()
+        input_val = input
 
-        starting_agent = (
-            args[0] if len(args) > 0 else kwargs.get("starting_agent")
+        update_trace_attributes(
+            input=input_val,
+            name=name,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            user_id=user_id,
+            metric_collection=metric_collection,
+            metrics=metrics,
         )
-        run_config: RunConfig | None = kwargs.get("run_config")
-        if run_config is None:
-            run_config = RunConfig()
-            kwargs["run_config"] = run_config
-
-        if run_config.model_provider is not None:
-            run_config.model_provider = _ObservedProvider(
-                run_config.model_provider,
-                metrics=getattr(starting_agent, "metrics", None) or metrics,
-                metric_collection=getattr(
-                    starting_agent, "metric_collection", None
-                )
-                or metric_collection,
-                deepeval_prompt=getattr(
-                    starting_agent, "deepeval_prompt", None
-                ),
-            )
 
-        input_val = args[1] if len(args) >= 2 else kwargs.get("input", None)
         with Observer(
             span_type="custom",
             metric_collection=metric_collection,
@@ -104,11 +192,140 @@ class Runner(BaseRunner):
         ) as observer:
             current_span = current_span_context.get()
             current_trace = current_trace_context.get()
-            current_trace.input = input_val
             if current_span:
                 current_span.input = input_val
-            res = super().run_sync(*args, **kwargs)
-            current_trace.output = str(res)
-            observer.result = str(res)
+            res = super().run_sync(
+                starting_agent,
+                input,
+                context=context,
+                max_turns=max_turns,
+                hooks=hooks,
+                run_config=run_config,
+                previous_response_id=previous_response_id,
+                conversation_id=conversation_id,
+                session=session,
+                **kwargs,  # backwards compatibility
+            )
+            _output = None
+            if thread_id:
+                _output = res.final_output
+            else:
+                _output = str(res)
+            update_trace_attributes(output=_output)
+            observer.result = _output
 
         return res
+
+    @classmethod
+    def run_streamed(
+        cls,
+        starting_agent: Agent[TContext],
+        input: Union[str, list[TResponseInputItem]],
+        *,
+        context: Optional[TContext] = None,
+        max_turns: int = DEFAULT_MAX_TURNS,
+        hooks: Optional[RunHooks[TContext]] = None,
+        run_config: Optional[RunConfig] = None,
+        previous_response_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        session: Optional[Session] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        **kwargs,  # backwards compatibility
+    ) -> RunResultStreaming:
+        is_agents_available()
+        # Manually enter observer; we'll exit when streaming finishes
+        observer = Observer(
+            span_type="custom",
+            metric_collection=metric_collection,
+            metrics=metrics,
+            func_name="run_streamed",
+            function_kwargs={"input": input},
+        )
+        observer.__enter__()
+
+        update_trace_attributes(
+            input=input,
+            name=name,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            user_id=user_id,
+            metric_collection=metric_collection,
+            metrics=metrics,
+        )
+
+        current_span = current_span_context.get()
+        if current_span:
+            current_span.input = input
+
+        res = super().run_streamed(
+            starting_agent,
+            input,
+            context=context,
+            max_turns=max_turns,
+            hooks=hooks,
+            run_config=run_config,
+            previous_response_id=previous_response_id,
+            conversation_id=conversation_id,
+            session=session,
+            **kwargs,  # backwards compatibility
+        )
+
+        # Runtime-patch stream_events so the observer closes only after streaming completes
+        orig_stream_events = res.stream_events
+
+        async def _patched_stream_events(self: RunResultStreaming):
+            try:
+                async for event in orig_stream_events():
+                    yield event
+                observer.result = self.final_output
+                update_trace_attributes(output=self.final_output)
+            except Exception as e:
+                observer.__exit__(type(e), e, e.__traceback__)
+                raise
+            finally:
+                observer.__exit__(None, None, None)
+
+        from types import MethodType as _MethodType
+
+        res.stream_events = _MethodType(_patched_stream_events, res)
+
+        return res
+
+
+def update_trace_attributes(
+    input: Any = None,
+    output: Any = None,
+    name: str = None,
+    tags: List[str] = None,
+    metadata: dict = None,
+    thread_id: str = None,
+    user_id: str = None,
+    metric_collection: str = None,
+    metrics: List[BaseMetric] = None,
+):
+    current_trace = current_trace_context.get()
+    if input:
+        current_trace.input = input
+    if output:
+        current_trace.output = output
+    if name:
+        current_trace.name = name
+    if tags:
+        current_trace.tags = tags
+    if metadata:
+        current_trace.metadata = metadata
+    if thread_id:
+        current_trace.thread_id = thread_id
+    if user_id:
+        current_trace.user_id = user_id
+    if metric_collection:
+        current_trace.metric_collection = metric_collection
+    if metrics:
+        current_trace.metrics = metrics

deepeval/tracing/tracing.py

@@ -8,9 +8,6 @@ import random
 import atexit
 import queue
 import uuid
-import os
-import json
-import time
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
@@ -496,6 +493,7 @@ class TraceManager:
                     asyncio.gather(*pending, return_exceptions=True)
                 )
             self.flush_traces(remaining_trace_request_bodies)
+            loop.run_until_complete(loop.shutdown_asyncgens())
             loop.close()
 
     def flush_traces(

{deepeval-3.5.4.dist-info → deepeval-3.5.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.4
+Version: 3.5.5
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -186,6 +186,8 @@ Let's pretend your LLM application is a RAG based customer support chatbot; here
 
 ## Installation
 
+Deepeval works with **Python>=3.9+**.
+
 ```
 pip install -U deepeval
 ```

{deepeval-3.5.4.dist-info → deepeval-3.5.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deepeval/__init__.py,sha256=6fsb813LD_jNhqR-xZnSdE5E-KsBbC3tc4oIg5ZMgTw,2115
-deepeval/_version.py,sha256=Vy_DqdUIdzt42W7BKglfMO5ghp2Wa6OV5Tatx__sA2U,27
+deepeval/_version.py,sha256=CJwAeAyMGnIxrkmBn8fpG6bwbVBsUZaTrtwbstM2LgA,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -141,24 +141,24 @@ deepeval/confident/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVG
 deepeval/confident/api.py,sha256=bOC71TaVAEgoXFtJ9yMo0-atmUUdBuvaclMGczMcR6o,8455
 deepeval/confident/types.py,sha256=-slFhDof_1maMgpLxqDRZv6kz6ZVY2hP_0uj_aveJKU,533
 deepeval/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/config/settings.py,sha256=h-hHrzTItfMkR1ZnMCF6xUeJ3DBwdOgO354QaEH-VaE,20861
+deepeval/config/settings.py,sha256=gRRi6nXEUKse13xAShU9MA18zo14vpIgl_R0xJ_0vnM,21314
 deepeval/config/settings_manager.py,sha256=PsBS_5dRJASak2AUDwjhjLSiezNz1fje0R3onoFCKC0,4014
 deepeval/config/utils.py,sha256=gSOVv18Tx1R72GucbdQesbZLFL-Y9EzbS4p7qd2w_xE,3799
-deepeval/constants.py,sha256=ZXIA1Hsjy03SvfaIx2hbj_oxHp-gJ-kYIyktnFD1AGg,1337
+deepeval/constants.py,sha256=Qe-es-WDPJndgBspEQXxddDCVanrAu03YWCpXsUkdo0,1368
 deepeval/dataset/__init__.py,sha256=rcum_VjBXu8eisCdr6sl84BgoZUs3x0tYbB2PnPtHGY,212
 deepeval/dataset/api.py,sha256=ZxkEqAF4nZH_Ys_1f5r9N2LFI_vBcAJxt8eJm7Mplpw,831
-deepeval/dataset/dataset.py,sha256=dkUDYtK1z9sDn6A-HOohoHHUobWuaCiuHEABLAfP4kQ,49396
+deepeval/dataset/dataset.py,sha256=T2rzGGKeCjIkkhXY0ofnWh13W6gjjdjat9uVHCmhGFI,49493
 deepeval/dataset/golden.py,sha256=T-rTk4Hw1tANx_Iimv977F6Y4QK3s5OIB4PecU5FJDM,2338
 deepeval/dataset/test_run_tracer.py,sha256=5CdpDvhzkEEBRyqWi6egocaxiN6IRS3XfbACxEQZQeM,2544
-deepeval/dataset/types.py,sha256=cwsAXeaxvd511Uc7-zQ0OxsrPPduCfUSKRbuziAwWhA,309
-deepeval/dataset/utils.py,sha256=kqzzM1kTtAmlt0kv4_6lLQ-oVR20PtcxdKE4NmoZFpM,5735
+deepeval/dataset/types.py,sha256=CWeOIBPK2WdmRUqjFa9gfN-w2da0r8Ilzl3ToDpJQoQ,558
+deepeval/dataset/utils.py,sha256=fGHqUxqBRGLp3ck1QHLuAQeqtkEERfi4-G5pSvh6KII,6606
 deepeval/errors.py,sha256=_K5wywEw2gp0DDECyeY6UrSI9GgOtCBbfPxt5maIzSY,113
 deepeval/evaluate/__init__.py,sha256=315IaMiYEz7oJhZ4kPTBfeCNd1xF-wWVU6KOQnrKQpE,291
 deepeval/evaluate/api.py,sha256=rkblH0ZFAAdyuF0Ymh7JE1pIJPR9yFuPrn9SQaCEQp4,435
 deepeval/evaluate/compare.py,sha256=tdSJY4E7YJ_zO3dzvpwngZHLiUI2YQcTWJOLI83htsQ,9855
 deepeval/evaluate/configs.py,sha256=QfWjaWNxLsgEe8-5j4PIs5WcSyEckiWt0qdpXSpl57M,928
 deepeval/evaluate/evaluate.py,sha256=NPAJ2iJqJI_RurXKUIC0tft_ozYMIKwZf5iPfmnNhQc,10412
-deepeval/evaluate/execute.py,sha256=fKMbH5rdfySTDUkvu808dLZ0NCwHQXMolz4ggmdZq90,79382
+deepeval/evaluate/execute.py,sha256=45m3w3QSAWVHRNTSqLZcpUI1bA_qRFWIGu292WKTjcA,87953
 deepeval/evaluate/types.py,sha256=IGZ3Xsj0UecPI3JNeTpJaK1gDvlepokfCmHwtItIW9M,831
 deepeval/evaluate/utils.py,sha256=kkliSGzuICeUsXDtlMMPfN95dUKlqarNhfciSffd4gI,23143
 deepeval/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -395,11 +395,11 @@ deepeval/openai/extractors.py,sha256=q062nlYKuPVwqfLFYCD1yWv7xHF1U_XrYdAp5ve2l_E
 deepeval/openai/patch.py,sha256=tPDqXaBScBJveM9P5xLT_mVwkubw0bOey-efvdjZIfg,7466
 deepeval/openai/utils.py,sha256=-84VZGUsnzRkYAFWc_DGaGuQTDCUItk0VtUTdjtSxg4,2748
 deepeval/openai_agents/__init__.py,sha256=u-e9laod3LyPfLcI5lr7Yhk8ArfWvlpr-D4_idWIt0A,321
-deepeval/openai_agents/agent.py,sha256=lgwc9pXhJn1xbytl2sy58aZX-gIsFP87fhImXnV9EJU,5564
-deepeval/openai_agents/callback_handler.py,sha256=AXnL9teO8txUwK3cxBpqFTki5ZWOEy6kSglLjrnCNyA,3264
+deepeval/openai_agents/agent.py,sha256=PYOhLELRXfGAP_fje70X3Ovm3WjF24mQYWdwrobwcr4,6173
+deepeval/openai_agents/callback_handler.py,sha256=-tOXJ3SMKqH5u41cB_g7FBjaX5qAuqVAaAv7vQtiBVc,3025
 deepeval/openai_agents/extractors.py,sha256=0jZxwgY1NQ3mMxVWPpLcMpKlbj-aYV7rwuzRzG8hdZs,11529
 deepeval/openai_agents/patch.py,sha256=zSmRV5yOReHC6IylhT93SM1nQpmH3sEWfYcJqa_iM84,3684
-deepeval/openai_agents/runner.py,sha256=eWXPrjDE7mE0hB7D1RXFCv3mDtHjNrWJV7Ba5QOTJb8,4173
+deepeval/openai_agents/runner.py,sha256=pRwe6DX6kpXia6btl4TAWlnXpk88MsQfM8yWkGufyk8,10608
 deepeval/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/plugins/plugin.py,sha256=_dwsdx4Dg9DbXxK3f7zJY4QWTJQWc7QE1HmIg2Zjjag,1515
 deepeval/progress_context.py,sha256=ZSKpxrE9sdgt9G3REKnVeXAv7GJXHHVGgLynpG1Pudw,3557
@@ -457,12 +457,12 @@ deepeval/tracing/otel/exporter.py,sha256=dXQd834zm5rm1ss9pWkBBlk-JSdtiw7aFLso2hM
 deepeval/tracing/otel/utils.py,sha256=g8yAzhqbPh1fOKCWkfNekC6AVotLfu1SUcfNMo6zii8,9786
 deepeval/tracing/patchers.py,sha256=DAPNkhrDtoeyJIVeQDUMhTz-xGcXu00eqjQZmov8FiU,3096
 deepeval/tracing/perf_epoch_bridge.py,sha256=iyAPddB6Op7NpMtPHJ29lDm53Btz9yLaN6xSCfTRQm4,1825
-deepeval/tracing/tracing.py,sha256=vOVFdN6fVMW53XhyqTZSfp4vI7DCqRez4TKNhdhr-sg,42277
+deepeval/tracing/tracing.py,sha256=b-0T3W6lAEOEGhODx0e-yIwBkm5V46EDNAWS9lcWkD0,42306
 deepeval/tracing/types.py,sha256=l_utWKerNlE5H3mOKpeUJLsvpP3cMyjH7HRANNgTmSQ,5306
 deepeval/tracing/utils.py,sha256=w_kdhuyBCygllnbqLpDdKJqpJo42t3ZMlGhNicV2A8c,6467
 deepeval/utils.py,sha256=r8tV_NYJSi6ib-oQw6cLw3L7ZSe4KIJVJc1ng6-kDX4,17179
-deepeval-3.5.4.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.5.4.dist-info/METADATA,sha256=fJ15yXxlzKTfOsoW5z7uxIJ4Qx6X-UTpKj7pabi5Tv8,18682
-deepeval-3.5.4.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.5.4.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
-deepeval-3.5.4.dist-info/RECORD,,
+deepeval-3.5.5.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.5.5.dist-info/METADATA,sha256=Js_9nOjXPh0YQOokcbPvquIW7lBRQMphrLHTYZ8-pAE,18721
+deepeval-3.5.5.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.5.5.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
+deepeval-3.5.5.dist-info/RECORD,,