deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/mlllms/gemini_model.py

@@ -1,284 +0,0 @@
-from typing import Optional, List, Union
-import requests
-from pydantic import BaseModel
-from google.genai import types
-from google import genai
-
-from deepeval.models.retry_policy import (
-    create_retry_decorator,
-)
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
-from deepeval.models.base_model import DeepEvalBaseMLLM
-from deepeval.test_case import MLLMImage
-from deepeval.config.settings import get_settings
-from deepeval.constants import ProviderSlug as PS
-
-
-default_multimodal_gemini_model = "gemini-1.5-pro"
-# consistent retry rules
-retry_gemini = create_retry_decorator(PS.GOOGLE)
-
-
-class MultimodalGeminiModel(DeepEvalBaseMLLM):
-    """Class that implements Google Gemini models for multimodal evaluation.
-
-    This class provides integration with Google's Gemini models through the Google GenAI SDK,
-    supporting both text and multimodal (text + image) inputs for evaluation tasks.
-    To use Gemini API, set api_key attribute only.
-    To use Vertex AI API, set project and location attributes.
-
-    Attributes:
-        model_name: Name of the Gemini model to use
-        api_key: Google API key for authentication
-        project: Google Cloud project ID
-        location: Google Cloud location
-
-    Example:
-        ```python
-        from deepeval.models import MultimodalGeminiModel
-
-        # Initialize the model
-        model = MultimodalGeminiModel(
-            model_name="gemini-pro-vision",
-            api_key="your-api-key"
-        )
-
-        # Generate text from text + image input
-        response = model.generate([
-            "Describe what you see in this image:",
-            MLLMImage(url="path/to/image.jpg", local=True)
-        ])
-        ```
-    """
-
-    def __init__(
-        self,
-        model_name: Optional[str] = None,
-        api_key: Optional[str] = None,
-        project: Optional[str] = None,
-        location: Optional[str] = None,
-        *args,
-        **kwargs,
-    ):
-        model_name = (
-            model_name
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GEMINI_MODEL_NAME)
-            or default_multimodal_gemini_model
-        )
-
-        # Get API key from key handler if not provided
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_API_KEY
-        )
-        self.project = project or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_PROJECT
-        )
-        self.location = location or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_LOCATION
-        )
-        self.use_vertexai = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_GENAI_USE_VERTEXAI
-        )
-
-        super().__init__(model_name, *args, **kwargs)
-        self.model = self.load_model(*args, **kwargs)
-
-    def should_use_vertexai(self):
-        """Checks if the model should use Vertex AI for generation.
-
-        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
-        environment variable. If not set, it checks for the presence of the
-        project and location.
-
-        Returns:
-            True if the model should use Vertex AI, False otherwise
-        """
-        if self.use_vertexai is not None:
-            return self.use_vertexai.lower() == "yes"
-
-        if self.project and self.location:
-            return True
-        else:
-            return False
-
-    def load_model(self, *args, **kwargs):
-        """Creates a client.
-        With Gen AI SDK, model is set at inference time, so there is no
-        model to load and initialize.
-        This method name is kept for compatibility with other LLMs.
-
-        Returns:
-            A GenerativeModel instance configured for evaluation.
-        """
-        if self.should_use_vertexai():
-            if not self.project or not self.location:
-                raise ValueError(
-                    "When using Vertex AI API, both project and location are required."
-                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
-                    "or set them in your DeepEval configuration."
-                )
-
-            # Create client for Vertex AI
-            self.client = genai.Client(
-                vertexai=True, project=self.project, location=self.location
-            )
-        else:
-            if not self.api_key:
-                raise ValueError(
-                    "Google API key is required. Either provide it directly, set GOOGLE_API_KEY environment variable, "
-                    "or set it in your DeepEval configuration."
-                )
-
-            # Create client for Gemini API
-            self.client = genai.Client(api_key=self.api_key)
-
-        # Configure default model generation settings
-        self.model_safety_settings = [
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-                threshold=types.HarmBlockThreshold.BLOCK_ONLY_HIGH,
-            ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
-                threshold=types.HarmBlockThreshold.BLOCK_ONLY_HIGH,
-            ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-                threshold=types.HarmBlockThreshold.BLOCK_ONLY_HIGH,
-            ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-                threshold=types.HarmBlockThreshold.BLOCK_ONLY_HIGH,
-            ),
-        ]
-        self.model_temperature = 0.0
-        return self.client.models
-
-    # TODO: Refactor genete prompt to minimize the work done on retry
-    @retry_gemini
-    def generate_prompt(
-        self, multimodal_input: List[Union[str, MLLMImage]] = []
-    ) -> List[Union[str, MLLMImage]]:
-        """Converts DeepEval multimodal input into GenAI SDK compatible format.
-
-        Args:
-            multimodal_input: List of strings and MLLMImage objects
-
-        Returns:
-            List of strings and PIL Image objects ready for model input
-
-        Raises:
-            ValueError: If an invalid input type is provided
-        """
-        prompt = []
-        settings = get_settings()
-
-        for ele in multimodal_input:
-            if isinstance(ele, str):
-                prompt.append(ele)
-            elif isinstance(ele, MLLMImage):
-                if ele.local:
-                    with open(ele.url, "rb") as f:
-                        image_data = f.read()
-                else:
-                    response = requests.get(
-                        ele.url,
-                        timeout=(
-                            settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
-                            settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
-                        ),
-                    )
-                    response.raise_for_status()
-                    image_data = response.content
-
-                image_part = types.Part.from_bytes(
-                    data=image_data, mime_type="image/jpeg"
-                )
-                prompt.append(image_part)
-            else:
-                raise ValueError(f"Invalid input type: {type(ele)}")
-        return prompt
-
-    @retry_gemini
-    def generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> str:
-        """Generates text from multimodal input.
-
-        Args:
-            multimodal_input: List of strings and MLLMImage objects
-            schema: Optional Pydantic model for structured output
-
-        Returns:
-            Generated text response
-        """
-        prompt = self.generate_prompt(multimodal_input)
-
-        if schema is not None:
-            response = self.client.models.generate_content(
-                model=self.model_name,
-                contents=prompt,
-                config=types.GenerateContentConfig(
-                    response_mime_type="application/json",
-                    response_schema=schema,
-                    safety_settings=self.model_safety_settings,
-                    temperature=self.model_temperature,
-                ),
-            )
-            return response.parsed, 0
-        else:
-            response = self.client.models.generate_content(
-                model=self.model_name,
-                contents=prompt,
-                config=types.GenerateContentConfig(
-                    safety_settings=self.model_safety_settings,
-                    temperature=self.model_temperature,
-                ),
-            )
-            return response.text, 0
-
-    @retry_gemini
-    async def a_generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> str:
-        """Asynchronously generates text from multimodal input.
-
-        Args:
-            multimodal_input: List of strings and MLLMImage objects
-            schema: Optional Pydantic model for structured output
-
-        Returns:
-            Generated text response
-        """
-        prompt = self.generate_prompt(multimodal_input)
-
-        if schema is not None:
-            response = await self.client.aio.models.generate_content(
-                model=self.model_name,
-                contents=prompt,
-                config=types.GenerateContentConfig(
-                    response_mime_type="application/json",
-                    response_schema=schema,
-                    safety_settings=self.model_safety_settings,
-                    temperature=self.model_temperature,
-                ),
-            )
-            return response.parsed, 0
-        else:
-            response = await self.client.aio.models.generate_content(
-                model=self.model_name,
-                contents=prompt,
-                config=types.GenerateContentConfig(
-                    safety_settings=self.model_safety_settings,
-                    temperature=self.model_temperature,
-                ),
-            )
-            return response.text, 0
-
-    def get_model_name(self) -> str:
-        """Returns the name of the Gemini model being used."""
-        return self.model_name

deepeval/models/mlllms/ollama_model.py

@@ -1,144 +0,0 @@
-from typing import Optional, Tuple, List, Union, Dict
-from ollama import Client, AsyncClient, ChatResponse
-from pydantic import BaseModel
-import requests
-import base64
-import io
-
-from deepeval.models.retry_policy import (
-    create_retry_decorator,
-)
-from deepeval.key_handler import KEY_FILE_HANDLER, ModelKeyValues
-from deepeval.models import DeepEvalBaseMLLM
-from deepeval.test_case import MLLMImage
-from deepeval.config.settings import get_settings
-from deepeval.constants import ProviderSlug as PS
-
-
-retry_ollama = create_retry_decorator(PS.OLLAMA)
-
-
-class MultimodalOllamaModel(DeepEvalBaseMLLM):
-    def __init__(self, **kwargs):
-        model_name = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        self.base_url = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_BASE_URL
-        )
-        self.kwargs = kwargs
-        super().__init__(model_name)
-
-    @retry_ollama
-    def generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> Tuple[Union[str, Dict], float]:
-        chat_model = self.load_model()
-        messages = self.generate_messages(multimodal_input)
-        response: ChatResponse = chat_model.chat(
-            model=self.model_name,
-            messages=messages,
-            format=schema.model_json_schema() if schema else None,
-        )
-        return (
-            (
-                schema.model_validate_json(response.message.content)
-                if schema
-                else response.message.content
-            ),
-            0,
-        )
-
-    @retry_ollama
-    async def a_generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> Tuple[str, float]:
-        chat_model = self.load_model(async_mode=True)
-        messages = self.generate_messages(multimodal_input)
-        response: ChatResponse = await chat_model.chat(
-            model=self.model_name,
-            messages=messages,
-            format=schema.model_json_schema() if schema else None,
-        )
-        return (
-            (
-                schema.model_validate_json(response.message.content)
-                if schema
-                else response.message.content
-            ),
-            0,
-        )
-
-    def generate_messages(
-        self, multimodal_input: List[Union[str, MLLMImage]] = []
-    ):
-        messages = []
-        for ele in multimodal_input:
-            if isinstance(ele, str):
-                messages.append(
-                    {
-                        "role": "user",
-                        "content": ele,
-                    }
-                )
-            elif isinstance(ele, MLLMImage):
-                img_b64 = self.convert_to_base64(ele.url, ele.local)
-                if img_b64 is not None:
-                    messages.append(
-                        {
-                            "role": "user",
-                            "images": [img_b64],
-                        }
-                    )
-        return messages
-
-    ###############################################
-    # Utilities
-    ###############################################
-
-    def convert_to_base64(self, image_source: str, is_local: bool) -> str:
-        from PIL import Image
-
-        settings = get_settings()
-        try:
-            if not is_local:
-                response = requests.get(
-                    image_source,
-                    stream=True,
-                    timeout=(
-                        settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
-                        settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
-                    ),
-                )
-                response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
-                image = Image.open(io.BytesIO(response.content))
-            else:
-                image = Image.open(image_source)
-
-            buffered = io.BytesIO()
-            image.save(buffered, format="JPEG")
-            img_str = base64.b64encode(buffered.getvalue()).decode()
-            return img_str
-
-        except (requests.exceptions.RequestException, OSError) as e:
-            # Log, then rethrow so @retry_ollama can retry generate_messages() on network failures
-            print(f"Image fetch/encode failed: {e}")
-            raise
-        except Exception as e:
-            print(f"Error converting image to base64: {e}")
-            return None
-
-    def load_model(self, async_mode: bool = False):
-        if not async_mode:
-            return self._build_client(Client)
-        return self._build_client(AsyncClient)
-
-    def _build_client(self, cls):
-        return cls(host=self.base_url, **self.kwargs)
-
-    def get_model_name(self):
-        return f"{self.model_name} (Ollama)"

deepeval/models/mlllms/openai_model.py

@@ -1,258 +0,0 @@
-from typing import Optional, Tuple, List, Union
-from openai import OpenAI, AsyncOpenAI
-from openai.types.chat import ParsedChatCompletion
-from pydantic import BaseModel
-from io import BytesIO
-import base64
-
-from deepeval.models.llms.openai_model import (
-    model_pricing,
-    structured_outputs_models,
-)
-from deepeval.models import DeepEvalBaseMLLM
-from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.test_case import MLLMImage
-from deepeval.models.utils import parse_model_name
-from deepeval.models.retry_policy import (
-    create_retry_decorator,
-    sdk_retries_for,
-)
-from deepeval.constants import ProviderSlug as PS
-
-
-retry_openai = create_retry_decorator(PS.OPENAI)
-
-valid_multimodal_gpt_models = [
-    "gpt-4o",
-    "gpt-4o-2024-05-13",
-    "gpt-4o-2024-08-06",
-    "gpt-4o-2024-11-20",
-    "gpt-4o-mini",
-    "gpt-4o-mini-2024-07-18",
-    "gpt-4.1",
-    "gpt-4.1-mini",
-    "gpt-4.1-nano",
-    "o1",
-    "o1-preview",
-    "o1-2024-12-17",
-    "o1-preview-2024-09-12",
-    "gpt-4.5-preview-2025-02-27",
-    "o4-mini",
-]
-
-default_multimodal_gpt_model = "gpt-4.1"
-
-unsupported_log_probs_multimodal_gpt_models = [
-    "o1",
-    "o1-preview",
-    "o1-2024-12-17",
-    "o1-preview-2024-09-12",
-    "gpt-4.5-preview-2025-02-27",
-    "o4-mini",
-]
-
-
-class MultimodalOpenAIModel(DeepEvalBaseMLLM):
-    def __init__(
-        self,
-        model: Optional[str] = None,
-        _openai_api_key: Optional[str] = None,
-        *args,
-        **kwargs,
-    ):
-        model_name = None
-        if isinstance(model, str):
-            model_name = parse_model_name(model)
-            if model_name not in valid_multimodal_gpt_models:
-                raise ValueError(
-                    f"Invalid model. Available Multimodal GPT models: {', '.join(model for model in valid_multimodal_gpt_models)}"
-                )
-        elif model is None:
-            model_name = default_multimodal_gpt_model
-
-        self._openai_api_key = _openai_api_key
-        self.args = args
-        self.kwargs = kwargs
-
-        super().__init__(model_name, *args, **kwargs)
-
-    ###############################################
-    # Generate functions
-    ###############################################
-
-    @retry_openai
-    def generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> Tuple[str, float]:
-        client = OpenAI(api_key=self._openai_api_key)
-        prompt = self.generate_prompt(multimodal_input)
-
-        if schema:
-            if self.model_name in structured_outputs_models:
-                messages = [{"role": "user", "content": prompt}]
-                response = client.beta.chat.completions.parse(
-                    model=self.model_name,
-                    messages=messages,
-                    response_format=schema,
-                )
-                input_tokens = response.usage.prompt_tokens
-                output_tokens = response.usage.completion_tokens
-                total_cost = self.calculate_cost(input_tokens, output_tokens)
-                generated_text = response.choices[0].message.parsed
-                return generated_text, total_cost
-
-        completion = client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
-        )
-        output = completion.choices[0].message.content
-        cost = self.calculate_cost(
-            completion.usage.prompt_tokens, completion.usage.completion_tokens
-        )
-        if schema:
-            json_output = trim_and_load_json(output)
-            return schema.model_validate(json_output), cost
-        else:
-            return output, cost
-
-    @retry_openai
-    async def a_generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> Tuple[str, float]:
-        client = AsyncOpenAI(api_key=self._openai_api_key)
-        prompt = self.generate_prompt(multimodal_input)
-
-        if schema:
-            if self.model_name in structured_outputs_models:
-                messages = [{"role": "user", "content": prompt}]
-                response = await client.beta.chat.completions.parse(
-                    model=self.model_name,
-                    messages=messages,
-                    response_format=schema,
-                )
-                input_tokens = response.usage.prompt_tokens
-                output_tokens = response.usage.completion_tokens
-                total_cost = self.calculate_cost(input_tokens, output_tokens)
-                generated_text = response.choices[0].message.parsed
-                return generated_text, total_cost
-
-        completion = await client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
-        )
-        output = completion.choices[0].message.content
-        cost = self.calculate_cost(
-            completion.usage.prompt_tokens, completion.usage.completion_tokens
-        )
-        if schema:
-            json_output = trim_and_load_json(output)
-            return schema.model_validate(json_output), cost
-        else:
-            return output, cost
-
-    ###############################################
-    # Other generate functions
-    ###############################################
-
-    @retry_openai
-    def generate_raw_response(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        top_logprobs: int = 5,
-    ) -> Tuple[ParsedChatCompletion, float]:
-        client = self._client()
-        prompt = self.generate_prompt(multimodal_input)
-        messages = [{"role": "user", "content": prompt}]
-        completion = client.chat.completions.create(
-            model=self.model_name,
-            messages=messages,
-            logprobs=True,
-            top_logprobs=top_logprobs,
-        )
-        # Cost calculation
-        input_tokens = completion.usage.prompt_tokens
-        output_tokens = completion.usage.completion_tokens
-        cost = self.calculate_cost(input_tokens, output_tokens)
-        return completion, cost
-
-    @retry_openai
-    async def a_generate_raw_response(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        top_logprobs: int = 5,
-    ) -> Tuple[ParsedChatCompletion, float]:
-        client = self._client(async_mode=True)
-        prompt = self.generate_prompt(multimodal_input)
-        messages = [{"role": "user", "content": prompt}]
-        completion = await client.chat.completions.create(
-            model=self.model_name,
-            messages=messages,
-            logprobs=True,
-            top_logprobs=top_logprobs,
-        )
-        # Cost calculation
-        input_tokens = completion.usage.prompt_tokens
-        output_tokens = completion.usage.completion_tokens
-        cost = self.calculate_cost(input_tokens, output_tokens)
-        return completion, cost
-
-    ###############################################
-    # Utilities
-    ###############################################
-
-    def generate_prompt(
-        self, multimodal_input: List[Union[str, MLLMImage]] = []
-    ):
-        prompt = []
-        for ele in multimodal_input:
-            if isinstance(ele, str):
-                prompt.append({"type": "text", "text": ele})
-            elif isinstance(ele, MLLMImage):
-                if ele.local:
-                    import PIL.Image
-
-                    image = PIL.Image.open(ele.url)
-                    visual_dict = {
-                        "type": "image_url",
-                        "image_url": {
-                            "url": f"data:image/jpeg;base64,{self.encode_pil_image(image)}"
-                        },
-                    }
-                else:
-                    visual_dict = {
-                        "type": "image_url",
-                        "image_url": {"url": ele.url},
-                    }
-                prompt.append(visual_dict)
-        return prompt
-
-    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        pricing = model_pricing.get(
-            self.model_name, model_pricing["gpt-4.1"]
-        )  # Default to 'gpt-4.1' if model not found
-        input_cost = input_tokens * pricing["input"]
-        output_cost = output_tokens * pricing["output"]
-        return input_cost + output_cost
-
-    def encode_pil_image(self, pil_image):
-        image_buffer = BytesIO()
-        if pil_image.mode in ("RGBA", "LA", "P"):
-            pil_image = pil_image.convert("RGB")
-        pil_image.save(image_buffer, format="JPEG")
-        image_bytes = image_buffer.getvalue()
-        base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
-        return base64_encoded_image
-
-    def _client(self, async_mode: bool = False):
-        kw = {"api_key": self._openai_api_key}
-        if not sdk_retries_for(PS.OPENAI):
-            kw["max_retries"] = 0
-        Client = AsyncOpenAI if async_mode else OpenAI
-        return Client(**kw)
-
-    def get_model_name(self):
-        return self.model_name