deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/models/embedding_models/local_embedding_model.py

@@ -1,5 +1,5 @@
 from openai import OpenAI, AsyncOpenAI
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from deepeval.key_handler import EmbeddingKeyValues, KEY_FILE_HANDLER
 from deepeval.models import DeepEvalBaseEmbeddingModel
@@ -15,25 +15,32 @@ retry_local = create_retry_decorator(PS.LOCAL)
 
 
 class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
-    def __init__(self, **kwargs):
-        self.base_url = KEY_FILE_HANDLER.fetch_data(
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        model: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **client_kwargs,
+    ):
+        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
+            EmbeddingKeyValues.LOCAL_EMBEDDING_API_KEY
+        )
+        self.base_url = base_url or KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_BASE_URL
         )
-        model_name = KEY_FILE_HANDLER.fetch_data(
+        self.model_name = model or KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_MODEL_NAME
         )
-        self.api_key = KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_API_KEY
-        )
-        self.kwargs = kwargs
-        super().__init__(model_name)
+        self.client_kwargs = client_kwargs or {}
+        self.generation_kwargs = generation_kwargs or {}
+        super().__init__(self.model_name)
 
     @retry_local
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
-            model=self.model_name,
-            input=[text],
+            model=self.model_name, input=[text], **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -41,8 +48,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
-            model=self.model_name,
-            input=texts,
+            model=self.model_name, input=texts, **self.generation_kwargs
         )
         return [data.embedding for data in response.data]
 
@@ -50,8 +56,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
-            model=self.model_name,
-            input=[text],
+            model=self.model_name, input=[text], **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -59,8 +64,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
-            model=self.model_name,
-            input=texts,
+            model=self.model_name, input=texts, **self.generation_kwargs
         )
         return [data.embedding for data in response.data]
 
@@ -76,27 +80,21 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
             return self._build_client(OpenAI)
         return self._build_client(AsyncOpenAI)
 
-    def _client_kwargs(self) -> Dict:
-        """
-        If Tenacity manages retries, turn off OpenAI SDK retries to avoid double retrying.
-        If users opt into SDK retries via DEEPEVAL_SDK_RETRY_PROVIDERS=local, leave them enabled.
-        """
-        kwargs = dict(self.kwargs or {})
+    def _build_client(self, cls):
+        client_kwargs = self.client_kwargs.copy()
         if not sdk_retries_for(PS.LOCAL):
-            kwargs["max_retries"] = 0
-        return kwargs
+            client_kwargs["max_retries"] = 0
 
-    def _build_client(self, cls):
-        kw = dict(
+        client_init_kwargs = dict(
             api_key=self.api_key,
             base_url=self.base_url,
-            **self._client_kwargs(),
+            **client_kwargs,
         )
         try:
-            return cls(**kw)
+            return cls(**client_init_kwargs)
         except TypeError as e:
-            # Older OpenAI SDKs may not accept max_retries; drop and retry once.
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
             if "max_retries" in str(e):
-                kw.pop("max_retries", None)
-                return cls(**kw)
+                client_init_kwargs.pop("max_retries", None)
+                return cls(**client_init_kwargs)
             raise

deepeval/models/embedding_models/ollama_embedding_model.py

@@ -1,5 +1,5 @@
 from ollama import Client, AsyncClient
-from typing import List
+from typing import List, Optional, Dict
 
 from deepeval.key_handler import EmbeddingKeyValues, KEY_FILE_HANDLER
 from deepeval.models import DeepEvalBaseEmbeddingModel
@@ -13,27 +13,28 @@ retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
-    def __init__(self, *args, **kwargs):
-        self.base_url = KEY_FILE_HANDLER.fetch_data(
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        host: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **client_kwargs,
+    ):
+        self.host = host or KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_BASE_URL
         )
-        model_name = KEY_FILE_HANDLER.fetch_data(
+        self.model_name = model or KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_MODEL_NAME
         )
-        # TODO: This is not being used. Clean it up in consistency PR
-        self.api_key = KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.LOCAL_EMBEDDING_API_KEY
-        )
-        self.args = args
-        self.kwargs = kwargs
-        super().__init__(model_name)
+        self.client_kwargs = client_kwargs or {}
+        self.generation_kwargs = generation_kwargs or {}
+        super().__init__(self.model_name)
 
     @retry_ollama
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
-            model=self.model_name,
-            input=text,
+            model=self.model_name, input=text, **self.generation_kwargs
         )
         return response["embeddings"][0]
 
@@ -41,8 +42,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
-            model=self.model_name,
-            input=texts,
+            model=self.model_name, input=texts, **self.generation_kwargs
         )
         return response["embeddings"]
 
@@ -50,8 +50,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
-            model=self.model_name,
-            input=text,
+            model=self.model_name, input=text, **self.generation_kwargs
         )
         return response["embeddings"][0]
 
@@ -59,8 +58,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
-            model=self.model_name,
-            input=texts,
+            model=self.model_name, input=texts, **self.generation_kwargs
         )
         return response["embeddings"]
 
@@ -74,7 +72,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         return self._build_client(AsyncClient)
 
     def _build_client(self, cls):
-        return cls(host=self.base_url, **self.kwargs)
+        return cls(host=self.host, **self.client_kwargs)
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"

deepeval/models/embedding_models/openai_embedding_model.py

@@ -19,27 +19,28 @@ default_openai_embedding_model = "text-embedding-3-small"
 
 
 class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
+
     def __init__(
         self,
         model: Optional[str] = None,
-        _openai_api_key: Optional[str] = None,
-        **kwargs,
+        openai_api_key: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **client_kwargs,
     ):
-        model_name = model if model else default_openai_embedding_model
-        if model_name not in valid_openai_embedding_models:
+        self.openai_api_key = openai_api_key
+        self.model_name = model if model else default_openai_embedding_model
+        if self.model_name not in valid_openai_embedding_models:
             raise ValueError(
                 f"Invalid model. Available OpenAI Embedding models: {', '.join(valid_openai_embedding_models)}"
             )
-        self._openai_api_key = _openai_api_key
-        self.model_name = model_name
-        self.kwargs = kwargs
+        self.client_kwargs = client_kwargs or {}
+        self.generation_kwargs = generation_kwargs or {}
 
     @retry_openai
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=text,
-            model=self.model_name,
+            input=text, model=self.model_name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -47,8 +48,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=texts,
-            model=self.model_name,
+            input=texts, model=self.model_name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -56,8 +56,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=text,
-            model=self.model_name,
+            input=text, model=self.model_name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -65,8 +64,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=texts,
-            model=self.model_name,
+            input=texts, model=self.model_name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -82,27 +80,20 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
             return self._build_client(OpenAI)
         return self._build_client(AsyncOpenAI)
 
-    def _client_kwargs(self) -> Dict:
-        """
-        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
-        If the user opts into SDK retries for 'openai' via DEEPEVAL_SDK_RETRY_PROVIDERS,
-        leave their retry settings as is.
-        """
-        kwargs = dict(self.kwargs or {})
+    def _build_client(self, cls):
+        client_kwargs = self.client_kwargs.copy()
         if not sdk_retries_for(PS.OPENAI):
-            kwargs["max_retries"] = 0
-        return kwargs
+            client_kwargs["max_retries"] = 0
 
-    def _build_client(self, cls):
-        kw = dict(
-            api_key=self._openai_api_key,
-            **self._client_kwargs(),
+        client_init_kwargs = dict(
+            api_key=self.openai_api_key,
+            **client_kwargs,
         )
         try:
-            return cls(**kw)
+            return cls(**client_init_kwargs)
         except TypeError as e:
             # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
             if "max_retries" in str(e):
-                kw.pop("max_retries", None)
-                return cls(**kw)
+                client_init_kwargs.pop("max_retries", None)
+                return cls(**client_init_kwargs)
             raise

deepeval/models/llms/grok_model.py

@@ -56,8 +56,8 @@ model_pricing = {
 class GrokModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        api_key: Optional[str] = None,
         model: Optional[str] = None,
+        api_key: Optional[str] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,

deepeval/models/llms/openai_model.py

@@ -70,6 +70,8 @@ unsupported_log_probs_gpt_models = [
     "o1-mini-2024-09-12",
     "o3-mini",
     "o3-mini-2025-01-31",
+    "o4-mini",
+    "o4-mini-2025-04-16",
     "gpt-4.5-preview-2025-02-27",
     "gpt-5",
     "gpt-5-2025-08-07",

deepeval/openai/__init__.py

@@ -1,37 +1,19 @@
-from importlib.machinery import SourceFileLoader
-import importlib.util
-import sys
-
-from deepeval.openai.patch import patch_openai
+try:
+    import openai  # noqa: F401
+except ImportError:
+    raise ModuleNotFoundError(
+        "Please install OpenAI to use this feature: 'pip install openai'"
+    )
 
 
-def load_and_patch_openai():
-    openai_spec = importlib.util.find_spec("openai")
-    if not openai_spec or not openai_spec.origin:
-        raise ImportError("Could not find the OpenAI package")
-    package_dirs = openai_spec.submodule_search_locations
-    loader = SourceFileLoader("deepeval_openai", openai_spec.origin)
-    new_spec = importlib.util.spec_from_loader(
-        "deepeval_openai",
-        loader,
-        origin=openai_spec.origin,
-        is_package=True,
-    )
-    deepeval_openai = importlib.util.module_from_spec(new_spec)
-    deepeval_openai.__path__ = package_dirs
-    sys.modules["deepeval_openai"] = deepeval_openai
-    loader.exec_module(deepeval_openai)
-    patch_openai(deepeval_openai)
-    return deepeval_openai
+try:
+    from openai import OpenAI, AsyncOpenAI  # noqa: F401
+except ImportError:
+    OpenAI = None  # type: ignore
+    AsyncOpenAI = None  # type: ignore
 
 
-patched_openai = load_and_patch_openai()
-openai = patched_openai
-OpenAI = patched_openai.OpenAI
-AsyncOpenAI = patched_openai.AsyncOpenAI
+if OpenAI or AsyncOpenAI:
+    from deepeval.openai.patch import patch_openai_classes
 
-__all__ = [
-    "openai",
-    "OpenAI",
-    "AsyncOpenAI",
-]
+    patch_openai_classes()

deepeval/openai/extractors.py

@@ -1,39 +1,36 @@
+import json
 from openai.types.chat import ChatCompletion, ParsedChatCompletion
-from typing import Optional, Union, List, Dict
+from typing import Any, Union, Dict
 from openai.types.responses import Response
-from pydantic import BaseModel
-import json
 
 from deepeval.test_case.llm_test_case import ToolCall
-
-
-class InputParameters(BaseModel):
-    model: Optional[str] = None
-    input: Optional[str] = None
-    instructions: Optional[str] = None
-    messages: Optional[List[Dict]] = None
-    tool_descriptions: Optional[Dict[str, str]] = None
-
-
-class OutputParameters(BaseModel):
-    output: Optional[str] = None
-    prompt_tokens: Optional[int] = None
-    completion_tokens: Optional[int] = None
-    tools_called: Optional[List[ToolCall]] = None
-
-
-def extract_input_parameters(
-    is_completion: bool, kwargs: Dict
+from deepeval.openai.utils import (
+    render_response_input,
+    stringify_multimodal_content,
+    render_messages,
+)
+from deepeval.openai.types import InputParameters, OutputParameters
+from deepeval.tracing.types import Message
+
+
+# guarding against errors to be compatible with legacy APIs
+def safe_extract_input_parameters(
+    is_completion: bool, kwargs: Dict[str, Any]
 ) -> InputParameters:
-    if is_completion:
-        return extract_input_parameters_from_completion(kwargs)
-    else:
-        return extract_input_parameters_from_response(kwargs)
+    try:
+        if is_completion:
+            return extract_input_parameters_from_completion(kwargs)
+        else:
+            return extract_input_parameters_from_response(kwargs)
+    except:
+        return InputParameters(model="NA")
 
 
-def extract_input_parameters_from_completion(kwargs: Dict) -> InputParameters:
+def extract_input_parameters_from_completion(
+    kwargs: Dict[str, Any],
+) -> InputParameters:
     model = kwargs.get("model")
-    messages = kwargs.get("messages")
+    messages = kwargs.get("messages") or []
     tools = kwargs.get("tools")
     tool_descriptions_map = (
         {
@@ -45,7 +42,7 @@ def extract_input_parameters_from_completion(kwargs: Dict) -> InputParameters:
     )
 
     # extract first user input from messages
-    input = ""
+    input_arg = ""
     user_messages = []
     for message in messages:
         role = message["role"]
@@ -53,20 +50,25 @@ def extract_input_parameters_from_completion(kwargs: Dict) -> InputParameters:
         if role == "user":
             user_messages.append(content)
     if len(user_messages) > 0:
-        input = user_messages[0]
+        input_arg = user_messages[0]
+
+    # render messages
+    messages = render_messages(messages)
 
     return InputParameters(
         model=model,
-        input=input,
+        input=stringify_multimodal_content(input_arg),
         messages=messages,
         tools=tools,
         tool_descriptions=tool_descriptions_map,
     )
 
 
-def extract_input_parameters_from_response(kwargs: Dict) -> InputParameters:
+def extract_input_parameters_from_response(
+    kwargs: Dict[str, Any],
+) -> InputParameters:
     model = kwargs.get("model")
-    input = kwargs.get("input")
+    input_payload = kwargs.get("input")
     instructions = kwargs.get("instructions")
     tools = kwargs.get("tools")
     tool_descriptions = (
@@ -74,35 +76,59 @@ def extract_input_parameters_from_response(kwargs: Dict) -> InputParameters:
         if tools is not None
         else None
     )
+    messages = []
+    if isinstance(input_payload, list):
+        messages = render_response_input(input_payload)
+    elif isinstance(input_payload, str):
+        messages = [
+            {
+                "role": "user",
+                "content": input_payload,
+            }
+        ]
+    if instructions:
+        messages.insert(
+            0,
+            {
+                "role": "system",
+                "content": instructions,
+            },
+        )
     return InputParameters(
         model=model,
-        input=input,
+        input=stringify_multimodal_content(input_payload),
+        messages=messages,
         instructions=instructions,
         tools=tools,
         tool_descriptions=tool_descriptions,
     )
 
 
-def extract_output_parameters(
+def safe_extract_output_parameters(
     is_completion: bool,
     response: Union[ChatCompletion, ParsedChatCompletion, Response],
     input_parameters: InputParameters,
 ) -> OutputParameters:
-    if is_completion:
-        return extract_output_parameters_from_completion(
-            response, input_parameters
-        )
-    else:
-        return extract_output_parameters_from_response(
-            response, input_parameters
-        )
+
+    # guarding against errors to be compatible with legacy APIs
+    try:
+        if is_completion:
+            return extract_output_parameters_from_completion(
+                response, input_parameters
+            )
+        else:
+            return extract_output_parameters_from_response(
+                response, input_parameters
+            )
+    except:
+        return OutputParameters()
 
 
 def extract_output_parameters_from_completion(
     completion: Union[ChatCompletion, ParsedChatCompletion],
     input_parameters: InputParameters,
 ) -> OutputParameters:
-    output = str(completion.choices[0].message.content)
+    output = str(completion.choices[0].message.content or "")
     prompt_tokens = completion.usage.prompt_tokens
     completion_tokens = completion.usage.completion_tokens
 
@@ -112,16 +138,21 @@ def extract_output_parameters_from_completion(
     if openai_tool_calls is not None:
         tools_called = []
         for tool_call in openai_tool_calls:
+            tool_descriptions = input_parameters.tool_descriptions or {}
             tools_called.append(
                 ToolCall(
                     name=tool_call.function.name,
                     input_parameters=json.loads(tool_call.function.arguments),
-                    description=input_parameters.tool_descriptions.get(
-                        tool_call.function.name
-                    ),
+                    description=tool_descriptions.get(tool_call.function.name),
                 )
             )
 
+    if not output and tools_called:
+        tool_calls = []
+        for tool_call in tools_called:
+            tool_calls.append(tool_call)
+        output = tool_calls
+
     return OutputParameters(
         output=output,
         prompt_tokens=prompt_tokens,
@@ -145,15 +176,19 @@ def extract_output_parameters_from_response(
         for tool_call in openai_raw_output:
             if tool_call.type != "function_call":
                 continue
+            tool_descriptions = input_parameters.tool_descriptions or {}
             tools_called.append(
                 ToolCall(
                     name=tool_call.name,
                     input_parameters=json.loads(tool_call.arguments),
-                    description=input_parameters.tool_descriptions.get(
-                        tool_call.name
-                    ),
+                    description=tool_descriptions.get(tool_call.name),
                 )
             )
+    if not output and tools_called:
+        tool_calls = []
+        for tool_call in tools_called:
+            tool_calls.append(tool_call)
+        output = tool_calls
 
     return OutputParameters(
         output=output,