deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/models/llms/litellm_model.py

@@ -9,10 +9,17 @@ from tenacity import (
     RetryCallState,
 )
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.utils import require_param
 
 
 def log_retry_error(retry_state: RetryCallState):
@@ -27,6 +34,10 @@ retryable_exceptions = (
     Exception,  # LiteLLM handles specific exceptions internally
 )
 
+_ALIAS_MAP = {
+    "base_url": ["api_base"],
+}
+
 
 class LiteLLMModel(DeepEvalBaseLLM):
     EXP_BASE: int = 2
@@ -39,24 +50,29 @@ class LiteLLMModel(DeepEvalBaseLLM):
         self,
         model: Optional[str] = None,
         api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
-        temperature: float = 0,
+        base_url: Optional[str] = None,
+        temperature: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-
         settings = get_settings()
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "LiteLLMModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+
         # Get model name from parameter or key file
-        model_name = model or settings.LITELLM_MODEL_NAME
-        if not model_name:
-            raise ValueError(
-                "Model name must be provided either through parameter or set-litellm command"
-            )
+        model = model or settings.LITELLM_MODEL_NAME
 
         # Get API key from parameter, or settings
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and aolike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = (
                 settings.LITELLM_API_KEY
@@ -67,8 +83,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
             )
 
         # Get API base from parameter, key file, or environment variable
-        self.api_base = (
-            api_base
+        base_url = (
+            base_url
             or (
                 str(settings.LITELLM_API_BASE)
                 if settings.LITELLM_API_BASE is not None
@@ -80,14 +96,37 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 else None
             )
         )
+        self.base_url = (
+            str(base_url).rstrip("/") if base_url is not None else None
+        )
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="LiteLLMModel",
+            env_var_name="LITELLM_MODEL_NAME",
+            param_hint="model",
+        )
 
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
         self.temperature = temperature
-        self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
         self.evaluation_cost = 0.0  # Initialize cost to 0.0
-        super().__init__(model_name)
+        super().__init__(model)
 
     @retry(
         wait=wait_exponential_jitter(
@@ -99,12 +138,19 @@ class LiteLLMModel(DeepEvalBaseLLM):
     )
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Union[str, Dict, Tuple[str, float]]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
         from litellm import completion
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         completion_params = {
-            "model": self.model_name,
-            "messages": [{"role": "user", "content": prompt}],
+            "model": self.name,
+            "messages": [{"role": "user", "content": content}],
             "temperature": self.temperature,
         }
 
@@ -112,12 +158,12 @@ class LiteLLMModel(DeepEvalBaseLLM):
             api_key = require_secret_api_key(
                 self.api_key,
                 provider_label="LiteLLM",
-                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                env_var_name="LITELLM_API_KEY|LITELLM_PROXY_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params["api_key"] = api_key
-        if self.api_base:
-            completion_params["api_base"] = self.api_base
+        if self.base_url:
+            completion_params["api_base"] = self.base_url
 
         # Add schema if provided
         if schema:
@@ -154,12 +200,19 @@ class LiteLLMModel(DeepEvalBaseLLM):
     )
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Union[str, Dict, Tuple[str, float]]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
         from litellm import acompletion
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         completion_params = {
-            "model": self.model_name,
-            "messages": [{"role": "user", "content": prompt}],
+            "model": self.name,
+            "messages": [{"role": "user", "content": content}],
             "temperature": self.temperature,
         }
 
@@ -171,8 +224,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params["api_key"] = api_key
-        if self.api_base:
-            completion_params["api_base"] = self.api_base
+        if self.base_url:
+            completion_params["api_base"] = self.base_url
 
         # Add schema if provided
         if schema:
@@ -221,12 +274,17 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
+            if check_if_multimodal(prompt):
+                prompt = convert_to_multi_modal_array(input=prompt)
+                content = self.generate_content(prompt)
+            else:
+                content = [{"type": "text", "text": prompt}]
             completion_params = {
-                "model": self.model_name,
-                "messages": [{"role": "user", "content": prompt}],
+                "model": self.name,
+                "messages": [{"role": "user", "content": content}],
                 "temperature": self.temperature,
                 "api_key": api_key,
-                "api_base": self.api_base,
+                "api_base": self.base_url,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
             }
@@ -262,12 +320,17 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
+            if check_if_multimodal(prompt):
+                prompt = convert_to_multi_modal_array(input=prompt)
+                content = self.generate_content(prompt)
+            else:
+                content = [{"type": "text", "text": prompt}]
             completion_params = {
-                "model": self.model_name,
-                "messages": [{"role": "user", "content": prompt}],
+                "model": self.name,
+                "messages": [{"role": "user", "content": content}],
                 "temperature": self.temperature,
                 "api_key": api_key,
-                "api_base": self.api_base,
+                "api_base": self.base_url,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
             }
@@ -302,12 +365,12 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to LiteLLMModel(...)",
             )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": temperature,
                 "n": n,
                 "api_key": api_key,
-                "api_base": self.api_base,
+                "api_base": self.base_url,
             }
             completion_params.update(self.kwargs)
 
@@ -320,6 +383,34 @@ class LiteLLMModel(DeepEvalBaseLLM):
             logging.error(f"Error in LiteLLM generate_samples: {e}")
             raise
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
+
     def calculate_cost(self, response: Any) -> float:
         """Calculate the cost of the response based on token usage."""
         try:
@@ -353,8 +444,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
     def get_model_name(self) -> str:
         from litellm import get_llm_provider
 
-        provider = get_llm_provider(self.model_name)
-        return f"{self.model_name} ({provider})"
+        provider = get_llm_provider(self.name)
+        return f"{self.name} ({provider})"
 
     def load_model(self, async_mode: bool = False):
         """
@@ -369,3 +460,6 @@ class LiteLLMModel(DeepEvalBaseLLM):
             None as LiteLLM handles client creation internally
         """
         return None
+
+    def supports_multimodal(self):
+        return True

deepeval/models/llms/local_model.py

@@ -1,17 +1,26 @@
-from typing import Optional, Tuple, Union, Dict
+from typing import Optional, Tuple, Union, Dict, List
 from pydantic import BaseModel, SecretStr
 from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ChatCompletion
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
+from deepeval.test_case import MLLMImage
+from deepeval.utils import (
+    check_if_multimodal,
+    convert_to_multi_modal_array,
+    require_param,
+)
 
 
 # consistent retry rules
@@ -22,47 +31,75 @@ class LocalModel(DeepEvalBaseLLM):
     def __init__(
         self,
         model: Optional[str] = None,
-        base_url: Optional[str] = None,
         api_key: Optional[str] = None,
-        temperature: float = 0,
+        base_url: Optional[str] = None,
+        temperature: Optional[float] = None,
         format: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
         settings = get_settings()
 
-        model_name = model or settings.LOCAL_MODEL_NAME
+        model = model or settings.LOCAL_MODEL_NAME
         if api_key is not None:
-            # keep it secret, keep it safe from serializings, logging and alike
-            self.local_model_api_key: SecretStr | None = SecretStr(api_key)
+            self.local_model_api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.local_model_api_key = settings.LOCAL_MODEL_API_KEY
 
+        base_url = (
+            base_url if base_url is not None else settings.LOCAL_MODEL_BASE_URL
+        )
         self.base_url = (
-            base_url
-            or settings.LOCAL_MODEL_BASE_URL
-            and str(settings.LOCAL_MODEL_BASE_URL)
+            str(base_url).rstrip("/") if base_url is not None else None
         )
         self.format = format or settings.LOCAL_MODEL_FORMAT
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="LocalModel",
+            env_var_name="LOCAL_MODEL_NAME",
+            param_hint="model",
+        )
+
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
         self.temperature = temperature
+
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
+        super().__init__(model)
 
     ###############################################
-    # Other generate functions
+    # Generate functions
     ###############################################
 
     @retry_local
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = prompt
+
         client = self.load_model(async_mode=False)
         response: ChatCompletion = client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
+            model=self.name,
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -77,11 +114,18 @@ class LocalModel(DeepEvalBaseLLM):
     @retry_local
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = prompt
+
         client = self.load_model(async_mode=True)
         response: ChatCompletion = await client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
+            model=self.name,
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -93,12 +137,72 @@ class LocalModel(DeepEvalBaseLLM):
         else:
             return res_content, 0.0
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        """
+        Converts multimodal input into OpenAI-compatible format.
+        Uses data URIs for all images since we can't guarantee local servers support URL fetching.
+        """
+        prompt = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                prompt.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                # For local servers, use data URIs for both remote and local images
+                # Most local servers don't support fetching external URLs
+                if element.url and not element.local:
+                    import requests
+                    import base64
+
+                    settings = get_settings()
+                    try:
+                        response = requests.get(
+                            element.url,
+                            timeout=(
+                                settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                                settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                            ),
+                        )
+                        response.raise_for_status()
+
+                        # Get mime type from response
+                        mime_type = response.headers.get(
+                            "content-type", element.mimeType or "image/jpeg"
+                        )
+
+                        # Encode to base64
+                        b64_data = base64.b64encode(response.content).decode(
+                            "utf-8"
+                        )
+                        data_uri = f"data:{mime_type};base64,{b64_data}"
+
+                    except Exception as e:
+                        raise ValueError(
+                            f"Failed to fetch remote image {element.url}: {e}"
+                        )
+                else:
+                    element.ensure_images_loaded()
+                    mime_type = element.mimeType or "image/jpeg"
+                    data_uri = f"data:{mime_type};base64,{element.dataBase64}"
+
+                prompt.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": data_uri},
+                    }
+                )
+        return prompt
+
     ###############################################
     # Model
     ###############################################
 
     def get_model_name(self):
-        return f"{self.model_name} (Local Model)"
+        return f"{self.name} (Local Model)"
+
+    def supports_multimodal(self):
+        return True
 
     def load_model(self, async_mode: bool = False):
         if not async_mode: