deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/llms/litellm_model.py

@@ -10,7 +10,10 @@ from tenacity import (
 )
 
 from deepeval.config.settings import get_settings
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
 
@@ -27,6 +30,10 @@ retryable_exceptions = (
     Exception,  # LiteLLM handles specific exceptions internally
 )
 
+_ALIAS_MAP = {
+    "base_url": ["api_base"],
+}
+
 
 class LiteLLMModel(DeepEvalBaseLLM):
     EXP_BASE: int = 2
@@ -39,16 +46,26 @@ class LiteLLMModel(DeepEvalBaseLLM):
         self,
         model: Optional[str] = None,
         api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
+        base_url: Optional[str] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
 
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "LiteLLMModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+
         settings = get_settings()
         # Get model name from parameter or key file
-        model_name = model or settings.LITELLM_MODEL_NAME
-        if not model_name:
+        model = model or settings.LITELLM_MODEL_NAME
+        if not model:
             raise ValueError(
                 "Model name must be provided either through parameter or set-litellm command"
             )
@@ -67,8 +84,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
             )
 
         # Get API base from parameter, key file, or environment variable
-        self.api_base = (
-            api_base
+        self.base_url = (
+            base_url
             or (
                 str(settings.LITELLM_API_BASE)
                 if settings.LITELLM_API_BASE is not None
@@ -84,10 +101,11 @@ class LiteLLMModel(DeepEvalBaseLLM):
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
-        self.kwargs = kwargs
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
         self.evaluation_cost = 0.0  # Initialize cost to 0.0
-        super().__init__(model_name)
+        super().__init__(model)
 
     @retry(
         wait=wait_exponential_jitter(
@@ -100,10 +118,11 @@ class LiteLLMModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Union[str, Dict, Tuple[str, float]]:
+
         from litellm import completion
 
         completion_params = {
-            "model": self.model_name,
+            "model": self.name,
             "messages": [{"role": "user", "content": prompt}],
             "temperature": self.temperature,
         }
@@ -116,8 +135,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params["api_key"] = api_key
-        if self.api_base:
-            completion_params["api_base"] = self.api_base
+        if self.base_url:
+            completion_params["api_base"] = self.base_url
 
         # Add schema if provided
         if schema:
@@ -155,10 +174,11 @@ class LiteLLMModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Union[str, Dict, Tuple[str, float]]:
+
         from litellm import acompletion
 
         completion_params = {
-            "model": self.model_name,
+            "model": self.name,
             "messages": [{"role": "user", "content": prompt}],
             "temperature": self.temperature,
         }
@@ -171,8 +191,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params["api_key"] = api_key
-        if self.api_base:
-            completion_params["api_base"] = self.api_base
+        if self.base_url:
+            completion_params["api_base"] = self.base_url
 
         # Add schema if provided
         if schema:
@@ -222,11 +242,11 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": self.temperature,
                 "api_key": api_key,
-                "api_base": self.api_base,
+                "api_base": self.base_url,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
             }
@@ -263,11 +283,11 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": self.temperature,
                 "api_key": api_key,
-                "api_base": self.api_base,
+                "api_base": self.base_url,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
             }
@@ -302,12 +322,12 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": temperature,
                 "n": n,
                 "api_key": api_key,
-                "api_base": self.api_base,
+                "api_base": self.base_url,
             }
             completion_params.update(self.kwargs)
 
@@ -353,8 +373,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
     def get_model_name(self) -> str:
         from litellm import get_llm_provider
 
-        provider = get_llm_provider(self.model_name)
-        return f"{self.model_name} ({provider})"
+        provider = get_llm_provider(self.name)
+        return f"{self.name} ({provider})"
 
     def load_model(self, async_mode: bool = False):
         """

deepeval/models/llms/local_model.py

@@ -9,7 +9,9 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
@@ -22,8 +24,8 @@ class LocalModel(DeepEvalBaseLLM):
     def __init__(
         self,
         model: Optional[str] = None,
-        base_url: Optional[str] = None,
         api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
         temperature: float = 0,
         format: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
@@ -31,7 +33,7 @@ class LocalModel(DeepEvalBaseLLM):
     ):
         settings = get_settings()
 
-        model_name = model or settings.LOCAL_MODEL_NAME
+        model = model or settings.LOCAL_MODEL_NAME
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
             self.local_model_api_key: SecretStr | None = SecretStr(api_key)
@@ -47,9 +49,10 @@ class LocalModel(DeepEvalBaseLLM):
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -59,9 +62,10 @@ class LocalModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=False)
         response: ChatCompletion = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             **self.generation_kwargs,
@@ -78,9 +82,10 @@ class LocalModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=True)
         response: ChatCompletion = await client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             **self.generation_kwargs,
@@ -98,7 +103,7 @@ class LocalModel(DeepEvalBaseLLM):
     ###############################################
 
     def get_model_name(self):
-        return f"{self.model_name} (Local Model)"
+        return f"{self.name} (Local Model)"
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:

deepeval/models/llms/ollama_model.py

@@ -1,14 +1,32 @@
-from ollama import Client, AsyncClient, ChatResponse
-from typing import Optional, Tuple, Union, Dict
+from typing import TYPE_CHECKING, Optional, Tuple, Union, Dict, List
 from pydantic import BaseModel
+import requests
+import base64
+import io
 
 from deepeval.config.settings import get_settings
+from deepeval.utils import require_dependency
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
+from deepeval.utils import convert_to_multi_modal_array, check_if_multimodal
+from deepeval.test_case import MLLMImage
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
+valid_multimodal_models = [
+    "llava:7b",
+    "llava:13b",
+    "llava:34b",
+    "llama4",
+    "gemma3",
+    "qwen3-vl",
+    "qwen2.5-vl",
+    # TODO: Add more models later on by looking at their catelogue
+]
+
+if TYPE_CHECKING:
+    from ollama import ChatResponse
 
 retry_ollama = create_retry_decorator(PS.OLLAMA)
 
@@ -23,7 +41,7 @@ class OllamaModel(DeepEvalBaseLLM):
         **kwargs,
     ):
         settings = get_settings()
-        model_name = model or settings.LOCAL_MODEL_NAME
+        model = model or settings.LOCAL_MODEL_NAME
         self.base_url = (
             base_url
             or (
@@ -35,10 +53,10 @@ class OllamaModel(DeepEvalBaseLLM):
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
-        # Raw kwargs destined for the underlying Ollama client
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -49,9 +67,17 @@ class OllamaModel(DeepEvalBaseLLM):
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
         chat_model = self.load_model()
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(prompt)
+            messages = self.generate_messages(prompt)
+        else:
+            messages = [{"role": "user", "content": prompt}]
+        print(messages)
+
         response: ChatResponse = chat_model.chat(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
+            model=self.name,
+            messages=messages,
             format=schema.model_json_schema() if schema else None,
             options={
                 **{"temperature": self.temperature},
@@ -72,9 +98,16 @@ class OllamaModel(DeepEvalBaseLLM):
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[str, float]:
         chat_model = self.load_model(async_mode=True)
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(prompt)
+            messages = self.generate_messages(prompt)
+        else:
+            messages = [{"role": "user", "content": prompt}]
+
         response: ChatResponse = await chat_model.chat(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
+            model=self.name,
+            messages=messages,
             format=schema.model_json_schema() if schema else None,
             options={
                 **{"temperature": self.temperature},
@@ -90,14 +123,78 @@ class OllamaModel(DeepEvalBaseLLM):
             0,
         )
 
+    def generate_messages(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        messages = []
+        for ele in multimodal_input:
+            if isinstance(ele, str):
+                messages.append(
+                    {
+                        "role": "user",
+                        "content": ele,
+                    }
+                )
+            elif isinstance(ele, MLLMImage):
+                img_b64 = self.convert_to_base64(ele.url, ele.local)
+                if img_b64 is not None:
+                    messages.append(
+                        {
+                            "role": "user",
+                            "images": [img_b64],
+                        }
+                    )
+        return messages
+
+    ###############################################
+    # Utilities
+    ###############################################
+
+    def convert_to_base64(self, image_source: str, is_local: bool) -> str:
+        from PIL import Image
+
+        settings = get_settings()
+        try:
+            if not is_local:
+                response = requests.get(
+                    image_source,
+                    stream=True,
+                    timeout=(
+                        settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                        settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                    ),
+                )
+                response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
+                image = Image.open(io.BytesIO(response.content))
+            else:
+                image = Image.open(image_source)
+
+            buffered = io.BytesIO()
+            image.save(buffered, format="JPEG")
+            img_str = base64.b64encode(buffered.getvalue()).decode()
+            return img_str
+
+        except (requests.exceptions.RequestException, OSError) as e:
+            # Log, then rethrow so @retry_ollama can retry generate_messages() on network failures
+            print(f"Image fetch/encode failed: {e}")
+            raise
+        except Exception as e:
+            print(f"Error converting image to base64: {e}")
+            return None
+
     ###############################################
     # Model
     ###############################################
 
     def load_model(self, async_mode: bool = False):
+        ollama = require_dependency(
+            "ollama",
+            provider_label="OllamaModel",
+            install_hint="Install it with `pip install ollama`.",
+        )
         if not async_mode:
-            return self._build_client(Client)
-        return self._build_client(AsyncClient)
+            return self._build_client(ollama.Client)
+        return self._build_client(ollama.AsyncClient)
 
     def _client_kwargs(self) -> Dict:
         """Return kwargs forwarded to the underlying Ollama Client/AsyncClient."""
@@ -110,5 +207,10 @@ class OllamaModel(DeepEvalBaseLLM):
         )
         return cls(**kw)
 
+    def supports_multimodal(self):
+        if self.name in valid_multimodal_models:
+            return True
+        return False
+
     def get_model_name(self):
-        return f"{self.model_name} (Ollama)"
+        return f"{self.name} (Ollama)"