deepeval 3.6.5__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/openai/extractors.py

@@ -1,29 +1,15 @@
+import json
 from openai.types.chat import ChatCompletion, ParsedChatCompletion
-from typing import Optional, Union, List, Dict
+from typing import Any, Union, Dict
 from openai.types.responses import Response
-from pydantic import BaseModel
-import json
 
 from deepeval.test_case.llm_test_case import ToolCall
-
-
-class InputParameters(BaseModel):
-    model: Optional[str] = None
-    input: Optional[str] = None
-    instructions: Optional[str] = None
-    messages: Optional[List[Dict]] = None
-    tool_descriptions: Optional[Dict[str, str]] = None
-
-
-class OutputParameters(BaseModel):
-    output: Optional[str] = None
-    prompt_tokens: Optional[int] = None
-    completion_tokens: Optional[int] = None
-    tools_called: Optional[List[ToolCall]] = None
+from deepeval.openai.utils import stringify_multimodal_content
+from deepeval.openai.types import InputParameters, OutputParameters
 
 
 def extract_input_parameters(
-    is_completion: bool, kwargs: Dict
+    is_completion: bool, kwargs: Dict[str, Any]
 ) -> InputParameters:
     if is_completion:
         return extract_input_parameters_from_completion(kwargs)
@@ -31,9 +17,11 @@ def extract_input_parameters(
         return extract_input_parameters_from_response(kwargs)
 
 
-def extract_input_parameters_from_completion(kwargs: Dict) -> InputParameters:
+def extract_input_parameters_from_completion(
+    kwargs: Dict[str, Any],
+) -> InputParameters:
     model = kwargs.get("model")
-    messages = kwargs.get("messages")
+    messages = kwargs.get("messages") or []
     tools = kwargs.get("tools")
     tool_descriptions_map = (
         {
@@ -45,7 +33,7 @@ def extract_input_parameters_from_completion(kwargs: Dict) -> InputParameters:
     )
 
     # extract first user input from messages
-    input = ""
+    input_arg = ""
     user_messages = []
     for message in messages:
         role = message["role"]
@@ -53,20 +41,22 @@ def extract_input_parameters_from_completion(kwargs: Dict) -> InputParameters:
         if role == "user":
             user_messages.append(content)
     if len(user_messages) > 0:
-        input = user_messages[0]
+        input_arg = user_messages[0]
 
     return InputParameters(
         model=model,
-        input=input,
+        input=stringify_multimodal_content(input_arg),
         messages=messages,
         tools=tools,
         tool_descriptions=tool_descriptions_map,
     )
 
 
-def extract_input_parameters_from_response(kwargs: Dict) -> InputParameters:
+def extract_input_parameters_from_response(
+    kwargs: Dict[str, Any],
+) -> InputParameters:
     model = kwargs.get("model")
-    input = kwargs.get("input")
+    input_payload = kwargs.get("input")
     instructions = kwargs.get("instructions")
     tools = kwargs.get("tools")
     tool_descriptions = (
@@ -74,9 +64,11 @@ def extract_input_parameters_from_response(kwargs: Dict) -> InputParameters:
         if tools is not None
         else None
     )
+    messages = input_payload if isinstance(input_payload, list) else None
     return InputParameters(
         model=model,
-        input=input,
+        input=stringify_multimodal_content(input_payload),
+        messages=messages,
         instructions=instructions,
         tools=tools,
         tool_descriptions=tool_descriptions,
@@ -102,7 +94,7 @@ def extract_output_parameters_from_completion(
     completion: Union[ChatCompletion, ParsedChatCompletion],
     input_parameters: InputParameters,
 ) -> OutputParameters:
-    output = str(completion.choices[0].message.content)
+    output = str(completion.choices[0].message.content or "")
     prompt_tokens = completion.usage.prompt_tokens
     completion_tokens = completion.usage.completion_tokens
 
@@ -112,13 +104,12 @@ def extract_output_parameters_from_completion(
     if openai_tool_calls is not None:
         tools_called = []
         for tool_call in openai_tool_calls:
+            tool_descriptions = input_parameters.tool_descriptions or {}
             tools_called.append(
                 ToolCall(
                     name=tool_call.function.name,
                     input_parameters=json.loads(tool_call.function.arguments),
-                    description=input_parameters.tool_descriptions.get(
-                        tool_call.function.name
-                    ),
+                    description=tool_descriptions.get(tool_call.function.name),
                 )
             )
 
@@ -145,13 +136,12 @@ def extract_output_parameters_from_response(
         for tool_call in openai_raw_output:
             if tool_call.type != "function_call":
                 continue
+            tool_descriptions = input_parameters.tool_descriptions or {}
             tools_called.append(
                 ToolCall(
                     name=tool_call.name,
                     input_parameters=json.loads(tool_call.arguments),
-                    description=input_parameters.tool_descriptions.get(
-                        tool_call.name
-                    ),
+                    description=tool_descriptions.get(tool_call.name),
                 )
             )
 

deepeval/openai/patch.py

@@ -1,204 +1,299 @@
-from typing import Callable, List, Optional
+from typing import Callable, List
 from functools import wraps
 
-from deepeval.openai.utils import (
-    get_attr_path,
-    set_attr_path,
-    add_test_case,
-    create_child_tool_spans,
-)
+
 from deepeval.openai.extractors import (
     extract_output_parameters,
     extract_input_parameters,
     InputParameters,
-    ToolCall,
+    OutputParameters,
+)
+from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import (
+    current_trace_context,
+    update_current_span,
+    update_llm_span,
 )
-from deepeval.tracing.context import update_current_span, update_llm_span
-from deepeval.tracing import trace_manager, observe
-from deepeval.metrics.base_metric import BaseMetric
-from deepeval.test_case import LLMTestCase
+from deepeval.tracing import observe
+from deepeval.tracing.trace_context import current_llm_context
+from deepeval.openai.utils import create_child_tool_spans
+
+# Store original methods for safety and potential unpatching
+_ORIGINAL_METHODS = {}
+_OPENAI_PATCHED = False
+
 
+def patch_openai_classes():
+    """Monkey patch OpenAI resource classes directly."""
+    global _OPENAI_PATCHED
 
-def patch_openai(openai_module):
-    if getattr(openai_module, "_deepeval_patched", False):
+    # Single guard - if already patched, return immediately
+    if _OPENAI_PATCHED:
         return
 
-    openai_module._deepeval_patched = True
-    openai_class = getattr(openai_module, "OpenAI", None)
-    async_openai_class = getattr(openai_module, "AsyncOpenAI", None)
-
-    if openai_class:
-        patch_openai_client(openai_class, is_async=False)
-    if async_openai_class:
-        patch_openai_client(async_openai_class, is_async=True)
-
-
-def patch_openai_client(openai_class, is_async: bool):
-    original_init = openai_class.__init__
-
-    @wraps(original_init)
-    def new_init(self, *args, **kwargs):
-        original_init(self, *args, **kwargs)
-        method_paths = {
-            # path → is_completion_method
-            "chat.completions.create": True,
-            "beta.chat.completions.parse": True,
-            "responses.create": False,
-        }
-        for path, is_completion in method_paths.items():
-            method = get_attr_path(self, path)
-            if not callable(method):
-                continue
-            if is_async:
-                patched_method = patch_async_openai_client_method(
-                    orig_method=method,
-                    is_completion_method=is_completion,
-                )
-            else:
-                patched_method = patch_sync_openai_client_method(
-                    orig_method=method,
-                    is_completion_method=is_completion,
-                )
-            set_attr_path(self, path, patched_method)
-
-    openai_class.__init__ = new_init
-
-
-def patch_async_openai_client_method(
+    try:
+        from openai.resources.chat.completions import (
+            Completions,
+            AsyncCompletions,
+        )
+
+        # Store original methods before patching
+        if hasattr(Completions, "create"):
+            _ORIGINAL_METHODS["Completions.create"] = Completions.create
+            Completions.create = _create_sync_wrapper(
+                Completions.create, is_completion_method=True
+            )
+
+        if hasattr(Completions, "parse"):
+            _ORIGINAL_METHODS["Completions.parse"] = Completions.parse
+            Completions.parse = _create_sync_wrapper(
+                Completions.parse, is_completion_method=True
+            )
+
+        if hasattr(AsyncCompletions, "create"):
+            _ORIGINAL_METHODS["AsyncCompletions.create"] = (
+                AsyncCompletions.create
+            )
+            AsyncCompletions.create = _create_async_wrapper(
+                AsyncCompletions.create, is_completion_method=True
+            )
+
+        if hasattr(AsyncCompletions, "parse"):
+            _ORIGINAL_METHODS["AsyncCompletions.parse"] = AsyncCompletions.parse
+            AsyncCompletions.parse = _create_async_wrapper(
+                AsyncCompletions.parse, is_completion_method=True
+            )
+
+    except ImportError:
+        pass
+
+    try:
+        from openai.resources.responses import Responses, AsyncResponses
+
+        if hasattr(Responses, "create"):
+            _ORIGINAL_METHODS["Responses.create"] = Responses.create
+            Responses.create = _create_sync_wrapper(
+                Responses.create, is_completion_method=False
+            )
+
+        if hasattr(AsyncResponses, "create"):
+            _ORIGINAL_METHODS["AsyncResponses.create"] = AsyncResponses.create
+            AsyncResponses.create = _create_async_wrapper(
+                AsyncResponses.create, is_completion_method=False
+            )
+
+    except ImportError:
+        pass
+
+    # Set flag at the END after successful patching
+    _OPENAI_PATCHED = True
+
+
+def _create_sync_wrapper(original_method, is_completion_method: bool):
+    """Create a wrapper for sync methods - called ONCE during patching."""
+
+    @wraps(original_method)
+    def method_wrapper(self, *args, **kwargs):
+        bound_method = original_method.__get__(self, type(self))
+        patched = _patch_sync_openai_client_method(
+            orig_method=bound_method, is_completion_method=is_completion_method
+        )
+        return patched(*args, **kwargs)
+
+    return method_wrapper
+
+
+def _create_async_wrapper(original_method, is_completion_method: bool):
+    """Create a wrapper for async methods - called ONCE during patching."""
+
+    @wraps(original_method)
+    async def method_wrapper(self, *args, **kwargs):
+        bound_method = original_method.__get__(self, type(self))
+        patched = _patch_async_openai_client_method(
+            orig_method=bound_method, is_completion_method=is_completion_method
+        )
+        return await patched(*args, **kwargs)
+
+    return method_wrapper
+
+
+def _patch_async_openai_client_method(
     orig_method: Callable,
     is_completion_method: bool = False,
 ):
     @wraps(orig_method)
-    async def patched_async_openai_method(
-        metrics: Optional[List[BaseMetric]] = None,
-        context: Optional[List[str]] = None,
-        retrieval_context: Optional[List[str]] = None,
-        expected_output: Optional[str] = None,
-        expected_tools: Optional[List[ToolCall]] = None,
-        *args,
-        **kwargs
-    ):
+    async def patched_async_openai_method(*args, **kwargs):
         input_parameters: InputParameters = extract_input_parameters(
             is_completion_method, kwargs
         )
-        is_traced = len(trace_manager.traces) > 0
-
-        if is_traced:
-
-            @observe(type="llm", model=input_parameters.model, metrics=metrics)
-            async def llm_generation(*args, **kwargs):
-                response = await orig_method(*args, **kwargs)
-                output_parameters = extract_output_parameters(
-                    is_completion_method, response, input_parameters
-                )
-                update_current_span(
-                    input=input_parameters.input
-                    or input_parameters.messages
-                    or "NA",
-                    output=output_parameters.output or "NA",
-                    expected_output=expected_output,
-                    retrieval_context=retrieval_context,
-                    context=context,
-                    tools_called=output_parameters.tools_called,
-                    expected_tools=expected_tools,
-                )
-                update_llm_span(
-                    input_token_count=output_parameters.prompt_tokens,
-                    output_token_count=output_parameters.completion_tokens,
-                )
-                create_child_tool_spans(output_parameters)
-                return response
-
-            return await llm_generation(*args, **kwargs)
-        else:
+
+        llm_context = current_llm_context.get()
+
+        @observe(
+            type="llm",
+            model=input_parameters.model,
+            metrics=llm_context.metrics,
+            metric_collection=llm_context.metric_collection,
+        )
+        async def llm_generation(*args, **kwargs):
             response = await orig_method(*args, **kwargs)
             output_parameters = extract_output_parameters(
                 is_completion_method, response, input_parameters
             )
-            test_case = LLMTestCase(
-                input=input_parameters.input,
-                actual_output=output_parameters.output,
-                expected_output=expected_output,
-                retrieval_context=retrieval_context,
-                context=context,
-                tools_called=output_parameters.tools_called,
-                expected_tools=expected_tools,
-            )
-            add_test_case(
-                test_case=test_case,
-                metrics=metrics,
-                input_parameters=input_parameters,
+            _update_all_attributes(
+                input_parameters,
+                output_parameters,
+                llm_context.expected_tools,
+                llm_context.expected_output,
+                llm_context.context,
+                llm_context.retrieval_context,
             )
+
             return response
 
+        return await llm_generation(*args, **kwargs)
+
     return patched_async_openai_method
 
 
-def patch_sync_openai_client_method(
+def _patch_sync_openai_client_method(
     orig_method: Callable,
     is_completion_method: bool = False,
 ):
     @wraps(orig_method)
-    def patched_sync_openai_method(
-        metrics: Optional[List[BaseMetric]] = None,
-        context: Optional[List[str]] = None,
-        retrieval_context: Optional[List[str]] = None,
-        expected_output: Optional[str] = None,
-        expected_tools: Optional[List[ToolCall]] = None,
-        *args,
-        **kwargs
-    ):
+    def patched_sync_openai_method(*args, **kwargs):
         input_parameters: InputParameters = extract_input_parameters(
             is_completion_method, kwargs
         )
-        is_traced = len(trace_manager.traces) > 0
-
-        if is_traced:
-
-            @observe(type="llm", model=input_parameters.model, metrics=metrics)
-            def llm_generation(*args, **kwargs):
-                response = orig_method(*args, **kwargs)
-                output_parameters = extract_output_parameters(
-                    is_completion_method, response, input_parameters
-                )
-                update_current_span(
-                    input=input_parameters.input
-                    or input_parameters.messages
-                    or "NA",
-                    output=output_parameters.output or "NA",
-                    expected_output=expected_output,
-                    retrieval_context=retrieval_context,
-                    context=context,
-                    tools_called=output_parameters.tools_called,
-                    expected_tools=expected_tools,
-                )
-                update_llm_span(
-                    input_token_count=output_parameters.prompt_tokens,
-                    output_token_count=output_parameters.completion_tokens,
-                )
-                create_child_tool_spans(output_parameters)
-                return response
-
-            return llm_generation(*args, **kwargs)
-        else:
+
+        llm_context = current_llm_context.get()
+
+        @observe(
+            type="llm",
+            model=input_parameters.model,
+            metrics=llm_context.metrics,
+            metric_collection=llm_context.metric_collection,
+        )
+        def llm_generation(*args, **kwargs):
             response = orig_method(*args, **kwargs)
             output_parameters = extract_output_parameters(
                 is_completion_method, response, input_parameters
             )
-            test_case = LLMTestCase(
-                input=input_parameters.input,
-                actual_output=output_parameters.output,
-                expected_output=expected_output,
-                retrieval_context=retrieval_context,
-                context=context,
-                tools_called=output_parameters.tools_called,
-                expected_tools=expected_tools,
-            )
-            add_test_case(
-                test_case=test_case,
-                metrics=metrics,
-                input_parameters=input_parameters,
+            _update_all_attributes(
+                input_parameters,
+                output_parameters,
+                llm_context.expected_tools,
+                llm_context.expected_output,
+                llm_context.context,
+                llm_context.retrieval_context,
             )
+
             return response
 
+        return llm_generation(*args, **kwargs)
+
     return patched_sync_openai_method
+
+
+def _update_all_attributes(
+    input_parameters: InputParameters,
+    output_parameters: OutputParameters,
+    expected_tools: List[ToolCall],
+    expected_output: str,
+    context: List[str],
+    retrieval_context: List[str],
+):
+    """Update span and trace attributes with input/output parameters."""
+    update_current_span(
+        input=input_parameters.input or input_parameters.messages or "NA",
+        output=output_parameters.output or "NA",
+        tools_called=output_parameters.tools_called,
+        # attributes to be added
+        expected_output=expected_output,
+        expected_tools=expected_tools,
+        context=context,
+        retrieval_context=retrieval_context,
+    )
+
+    llm_context = current_llm_context.get()
+
+    update_llm_span(
+        input_token_count=output_parameters.prompt_tokens,
+        output_token_count=output_parameters.completion_tokens,
+        prompt=llm_context.prompt,
+    )
+
+    if output_parameters.tools_called:
+        create_child_tool_spans(output_parameters)
+
+    __update_input_and_output_of_current_trace(
+        input_parameters, output_parameters
+    )
+
+
+def __update_input_and_output_of_current_trace(
+    input_parameters: InputParameters, output_parameters: OutputParameters
+):
+
+    current_trace = current_trace_context.get()
+    if current_trace:
+        if current_trace.input is None:
+            current_trace.input = (
+                input_parameters.input or input_parameters.messages
+            )
+
+        if current_trace.output is None:
+            current_trace.output = output_parameters.output
+
+    return
+
+
+def unpatch_openai_classes():
+    """Restore OpenAI resource classes to their original state."""
+    global _OPENAI_PATCHED
+
+    # If not patched, nothing to do
+    if not _OPENAI_PATCHED:
+        return
+
+    try:
+        from openai.resources.chat.completions import (
+            Completions,
+            AsyncCompletions,
+        )
+
+        # Restore original methods for Completions
+        if "Completions.create" in _ORIGINAL_METHODS:
+            Completions.create = _ORIGINAL_METHODS["Completions.create"]
+
+        if "Completions.parse" in _ORIGINAL_METHODS:
+            Completions.parse = _ORIGINAL_METHODS["Completions.parse"]
+
+        # Restore original methods for AsyncCompletions
+        if "AsyncCompletions.create" in _ORIGINAL_METHODS:
+            AsyncCompletions.create = _ORIGINAL_METHODS[
+                "AsyncCompletions.create"
+            ]
+
+        if "AsyncCompletions.parse" in _ORIGINAL_METHODS:
+            AsyncCompletions.parse = _ORIGINAL_METHODS["AsyncCompletions.parse"]
+
+    except ImportError:
+        pass
+
+    try:
+        from openai.resources.responses import Responses, AsyncResponses
+
+        # Restore original methods for Responses
+        if "Responses.create" in _ORIGINAL_METHODS:
+            Responses.create = _ORIGINAL_METHODS["Responses.create"]
+
+        # Restore original methods for AsyncResponses
+        if "AsyncResponses.create" in _ORIGINAL_METHODS:
+            AsyncResponses.create = _ORIGINAL_METHODS["AsyncResponses.create"]
+
+    except ImportError:
+        pass
+
+    # Reset the patched flag
+    _OPENAI_PATCHED = False

deepeval/openai/types.py

@@ -0,0 +1,20 @@
+from typing import Any, Optional, List, Dict
+from pydantic import BaseModel
+
+from deepeval.test_case.llm_test_case import ToolCall
+
+
+class InputParameters(BaseModel):
+    model: Optional[str] = None
+    input: Optional[str] = None
+    tools: Optional[List[Dict[str, Any]]] = None
+    instructions: Optional[str] = None
+    messages: Optional[List[Dict[str, Any]]] = None
+    tool_descriptions: Optional[Dict[str, str]] = None
+
+
+class OutputParameters(BaseModel):
+    output: Optional[str] = None
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    tools_called: Optional[List[ToolCall]] = None