deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/models/llms/litellm_model.py

@@ -9,13 +9,17 @@ from tenacity import (
     RetryCallState,
 )
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.utils import (
     require_secret_api_key,
     normalize_kwargs_and_extract_aliases,
 )
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.utils import require_param
 
 
 def log_retry_error(retry_state: RetryCallState):
@@ -47,11 +51,11 @@ class LiteLLMModel(DeepEvalBaseLLM):
         model: Optional[str] = None,
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
-        temperature: float = 0,
+        temperature: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-
+        settings = get_settings()
         normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
             "LiteLLMModel",
             kwargs,
@@ -62,18 +66,13 @@ class LiteLLMModel(DeepEvalBaseLLM):
         if base_url is None and "base_url" in alias_values:
             base_url = alias_values["base_url"]
 
-        settings = get_settings()
         # Get model name from parameter or key file
         model = model or settings.LITELLM_MODEL_NAME
-        if not model:
-            raise ValueError(
-                "Model name must be provided either through parameter or set-litellm command"
-            )
 
         # Get API key from parameter, or settings
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and aolike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = (
                 settings.LITELLM_API_KEY
@@ -84,7 +83,7 @@ class LiteLLMModel(DeepEvalBaseLLM):
             )
 
         # Get API base from parameter, key file, or environment variable
-        self.base_url = (
+        base_url = (
             base_url
             or (
                 str(settings.LITELLM_API_BASE)
@@ -97,13 +96,35 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 else None
             )
         )
+        self.base_url = (
+            str(base_url).rstrip("/") if base_url is not None else None
+        )
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="LiteLLMModel",
+            env_var_name="LITELLM_MODEL_NAME",
+            param_hint="model",
+        )
 
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
         self.temperature = temperature
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = normalized_kwargs
-        self.generation_kwargs = generation_kwargs or {}
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
         self.evaluation_cost = 0.0  # Initialize cost to 0.0
         super().__init__(model)
 
@@ -117,13 +138,19 @@ class LiteLLMModel(DeepEvalBaseLLM):
     )
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Union[str, Dict, Tuple[str, float]]:
+    ) -> Tuple[Union[str, BaseModel], float]:
 
         from litellm import completion
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         completion_params = {
             "model": self.name,
-            "messages": [{"role": "user", "content": prompt}],
+            "messages": [{"role": "user", "content": content}],
             "temperature": self.temperature,
         }
 
@@ -131,7 +158,7 @@ class LiteLLMModel(DeepEvalBaseLLM):
             api_key = require_secret_api_key(
                 self.api_key,
                 provider_label="LiteLLM",
-                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                env_var_name="LITELLM_API_KEY|LITELLM_PROXY_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params["api_key"] = api_key
@@ -173,13 +200,19 @@ class LiteLLMModel(DeepEvalBaseLLM):
     )
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Union[str, Dict, Tuple[str, float]]:
+    ) -> Tuple[Union[str, BaseModel], float]:
 
         from litellm import acompletion
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         completion_params = {
             "model": self.name,
-            "messages": [{"role": "user", "content": prompt}],
+            "messages": [{"role": "user", "content": content}],
             "temperature": self.temperature,
         }
 
@@ -241,9 +274,14 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
+            if check_if_multimodal(prompt):
+                prompt = convert_to_multi_modal_array(input=prompt)
+                content = self.generate_content(prompt)
+            else:
+                content = [{"type": "text", "text": prompt}]
             completion_params = {
                 "model": self.name,
-                "messages": [{"role": "user", "content": prompt}],
+                "messages": [{"role": "user", "content": content}],
                 "temperature": self.temperature,
                 "api_key": api_key,
                 "api_base": self.base_url,
@@ -251,6 +289,7 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 "top_logprobs": top_logprobs,
             }
             completion_params.update(self.kwargs)
+            completion_params.update(self.generation_kwargs)
 
             response = completion(**completion_params)
             cost = self.calculate_cost(response)
@@ -282,9 +321,14 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
+            if check_if_multimodal(prompt):
+                prompt = convert_to_multi_modal_array(input=prompt)
+                content = self.generate_content(prompt)
+            else:
+                content = [{"type": "text", "text": prompt}]
             completion_params = {
                 "model": self.name,
-                "messages": [{"role": "user", "content": prompt}],
+                "messages": [{"role": "user", "content": content}],
                 "temperature": self.temperature,
                 "api_key": api_key,
                 "api_base": self.base_url,
@@ -292,6 +336,7 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 "top_logprobs": top_logprobs,
             }
             completion_params.update(self.kwargs)
+            completion_params.update(self.generation_kwargs)
 
             response = await acompletion(**completion_params)
             cost = self.calculate_cost(response)
@@ -340,6 +385,34 @@ class LiteLLMModel(DeepEvalBaseLLM):
             logging.error(f"Error in LiteLLM generate_samples: {e}")
             raise
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
+
     def calculate_cost(self, response: Any) -> float:
         """Calculate the cost of the response based on token usage."""
         try:
@@ -389,3 +462,6 @@ class LiteLLMModel(DeepEvalBaseLLM):
             None as LiteLLM handles client creation internally
         """
         return None
+
+    def supports_multimodal(self):
+        return True

deepeval/models/llms/local_model.py

@@ -1,8 +1,9 @@
-from typing import Optional, Tuple, Union, Dict
+from typing import Optional, Tuple, Union, Dict, List
 from pydantic import BaseModel, SecretStr
 from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ChatCompletion
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
@@ -14,6 +15,12 @@ from deepeval.models.utils import (
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
+from deepeval.test_case import MLLMImage
+from deepeval.utils import (
+    check_if_multimodal,
+    convert_to_multi_modal_array,
+    require_param,
+)
 
 
 # consistent retry rules
@@ -26,7 +33,7 @@ class LocalModel(DeepEvalBaseLLM):
         model: Optional[str] = None,
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
-        temperature: float = 0,
+        temperature: Optional[float] = None,
         format: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
@@ -35,38 +42,64 @@ class LocalModel(DeepEvalBaseLLM):
 
         model = model or settings.LOCAL_MODEL_NAME
         if api_key is not None:
-            # keep it secret, keep it safe from serializings, logging and alike
-            self.local_model_api_key: SecretStr | None = SecretStr(api_key)
+            self.local_model_api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.local_model_api_key = settings.LOCAL_MODEL_API_KEY
 
+        base_url = (
+            base_url if base_url is not None else settings.LOCAL_MODEL_BASE_URL
+        )
         self.base_url = (
-            base_url
-            or settings.LOCAL_MODEL_BASE_URL
-            and str(settings.LOCAL_MODEL_BASE_URL)
+            str(base_url).rstrip("/") if base_url is not None else None
+        )
+        self.format = format or settings.LOCAL_MODEL_FORMAT or "json"
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="LocalModel",
+            env_var_name="LOCAL_MODEL_NAME",
+            param_hint="model",
         )
-        self.format = format or settings.LOCAL_MODEL_FORMAT
+
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
         self.temperature = temperature
-        # Keep sanitized kwargs for client call to strip legacy keys
+
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
         super().__init__(model)
 
     ###############################################
-    # Other generate functions
+    # Generate functions
     ###############################################
 
     @retry_local
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = prompt
 
         client = self.load_model(async_mode=False)
         response: ChatCompletion = client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -81,12 +114,18 @@ class LocalModel(DeepEvalBaseLLM):
     @retry_local
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = prompt
 
         client = self.load_model(async_mode=True)
         response: ChatCompletion = await client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -98,6 +137,63 @@ class LocalModel(DeepEvalBaseLLM):
         else:
             return res_content, 0.0
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        """
+        Converts multimodal input into OpenAI-compatible format.
+        Uses data URIs for all images since we can't guarantee local servers support URL fetching.
+        """
+        prompt = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                prompt.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                # For local servers, use data URIs for both remote and local images
+                # Most local servers don't support fetching external URLs
+                if element.url and not element.local:
+                    import requests
+                    import base64
+
+                    settings = get_settings()
+                    try:
+                        response = requests.get(
+                            element.url,
+                            timeout=(
+                                settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                                settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                            ),
+                        )
+                        response.raise_for_status()
+
+                        # Get mime type from response
+                        mime_type = response.headers.get(
+                            "content-type", element.mimeType or "image/jpeg"
+                        )
+
+                        # Encode to base64
+                        b64_data = base64.b64encode(response.content).decode(
+                            "utf-8"
+                        )
+                        data_uri = f"data:{mime_type};base64,{b64_data}"
+
+                    except Exception as e:
+                        raise ValueError(
+                            f"Failed to fetch remote image {element.url}: {e}"
+                        )
+                else:
+                    element.ensure_images_loaded()
+                    mime_type = element.mimeType or "image/jpeg"
+                    data_uri = f"data:{mime_type};base64,{element.dataBase64}"
+
+                prompt.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": data_uri},
+                    }
+                )
+        return prompt
+
     ###############################################
     # Model
     ###############################################
@@ -105,6 +201,9 @@ class LocalModel(DeepEvalBaseLLM):
     def get_model_name(self):
         return f"{self.name} (Local Model)"
 
+    def supports_multimodal(self):
+        return True
+
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(OpenAI)

deepeval/models/llms/ollama_model.py

@@ -1,11 +1,10 @@
 from typing import TYPE_CHECKING, Optional, Tuple, Union, Dict, List
 from pydantic import BaseModel
-import requests
 import base64
-import io
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
-from deepeval.utils import require_dependency
+from deepeval.utils import require_dependency, require_param
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
@@ -13,17 +12,7 @@ from deepeval.utils import convert_to_multi_modal_array, check_if_multimodal
 from deepeval.test_case import MLLMImage
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-
-valid_multimodal_models = [
-    "llava:7b",
-    "llava:13b",
-    "llava:34b",
-    "llama4",
-    "gemma3",
-    "qwen3-vl",
-    "qwen2.5-vl",
-    # TODO: Add more models later on by looking at their catelogue
-]
+from deepeval.models.llms.constants import OLLAMA_MODELS_DATA
 
 if TYPE_CHECKING:
     from ollama import ChatResponse
@@ -36,26 +25,46 @@ class OllamaModel(DeepEvalBaseLLM):
         self,
         model: Optional[str] = None,
         base_url: Optional[str] = None,
-        temperature: float = 0,
+        temperature: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
         settings = get_settings()
-        model = model or settings.LOCAL_MODEL_NAME
-        self.base_url = (
-            base_url
-            or (
-                settings.LOCAL_MODEL_BASE_URL
-                and str(settings.LOCAL_MODEL_BASE_URL)
-            )
-            or "http://localhost:11434"
+        model = model or settings.OLLAMA_MODEL_NAME
+        self.model_data = OLLAMA_MODELS_DATA.get(model)
+
+        if base_url is not None:
+            self.base_url = str(base_url).rstrip("/")
+        elif settings.LOCAL_MODEL_BASE_URL is not None:
+            self.base_url = str(settings.LOCAL_MODEL_BASE_URL).rstrip("/")
+        else:
+            self.base_url = "http://localhost:11434"
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="OllamaModel",
+            env_var_name="LOCAL_MODEL_NAME",
+            param_hint="model",
         )
+
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
         self.temperature = temperature
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
         super().__init__(model)
 
     ###############################################
@@ -65,7 +74,7 @@ class OllamaModel(DeepEvalBaseLLM):
     @retry_ollama
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
         chat_model = self.load_model()
 
         if check_if_multimodal(prompt):
@@ -73,7 +82,6 @@ class OllamaModel(DeepEvalBaseLLM):
             messages = self.generate_messages(prompt)
         else:
             messages = [{"role": "user", "content": prompt}]
-        print(messages)
 
         response: ChatResponse = chat_model.chat(
             model=self.name,
@@ -96,7 +104,7 @@ class OllamaModel(DeepEvalBaseLLM):
     @retry_ollama
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[str, float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
         chat_model = self.load_model(async_mode=True)
 
         if check_if_multimodal(prompt):
@@ -127,60 +135,78 @@ class OllamaModel(DeepEvalBaseLLM):
         self, multimodal_input: List[Union[str, MLLMImage]] = []
     ):
         messages = []
-        for ele in multimodal_input:
-            if isinstance(ele, str):
+
+        for element in multimodal_input:
+            if isinstance(element, str):
+                messages.append(
+                    {
+                        "role": "user",
+                        "content": element,
+                    }
+                )
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    import requests
+                    from PIL import Image
+                    import io
+
+                    settings = get_settings()
+                    try:
+                        response = requests.get(
+                            element.url,
+                            stream=True,
+                            timeout=(
+                                settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                                settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                            ),
+                        )
+                        response.raise_for_status()
+
+                        # Convert to JPEG and encode
+                        image = Image.open(io.BytesIO(response.content))
+                        buffered = io.BytesIO()
+
+                        # Convert RGBA/LA/P to RGB for JPEG
+                        if image.mode in ("RGBA", "LA", "P"):
+                            image = image.convert("RGB")
+
+                        image.save(buffered, format="JPEG")
+                        img_b64 = base64.b64encode(buffered.getvalue()).decode()
+
+                    except (requests.exceptions.RequestException, OSError) as e:
+                        print(f"Image fetch/encode failed: {e}")
+                        raise
+                else:
+                    element.ensure_images_loaded()
+                    img_b64 = element.dataBase64
+
                 messages.append(
                     {
                         "role": "user",
-                        "content": ele,
+                        "images": [img_b64],
                     }
                 )
-            elif isinstance(ele, MLLMImage):
-                img_b64 = self.convert_to_base64(ele.url, ele.local)
-                if img_b64 is not None:
-                    messages.append(
-                        {
-                            "role": "user",
-                            "images": [img_b64],
-                        }
-                    )
+
         return messages
 
     ###############################################
-    # Utilities
+    # Capabilities
     ###############################################
 
-    def convert_to_base64(self, image_source: str, is_local: bool) -> str:
-        from PIL import Image
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
 
-        settings = get_settings()
-        try:
-            if not is_local:
-                response = requests.get(
-                    image_source,
-                    stream=True,
-                    timeout=(
-                        settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
-                        settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
-                    ),
-                )
-                response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
-                image = Image.open(io.BytesIO(response.content))
-            else:
-                image = Image.open(image_source)
-
-            buffered = io.BytesIO()
-            image.save(buffered, format="JPEG")
-            img_str = base64.b64encode(buffered.getvalue()).decode()
-            return img_str
-
-        except (requests.exceptions.RequestException, OSError) as e:
-            # Log, then rethrow so @retry_ollama can retry generate_messages() on network failures
-            print(f"Image fetch/encode failed: {e}")
-            raise
-        except Exception as e:
-            print(f"Error converting image to base64: {e}")
-            return None
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        return self.model_data.supports_json
 
     ###############################################
     # Model
@@ -207,10 +233,5 @@ class OllamaModel(DeepEvalBaseLLM):
         )
         return cls(**kw)
 
-    def supports_multimodal(self):
-        if self.name in valid_multimodal_models:
-            return True
-        return False
-
     def get_model_name(self):
         return f"{self.name} (Ollama)"