deepeval 3.7.1__py3-none-any.whl → 3.7.3__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.7.1"
+__version__: str = "3.7.3"

deepeval/benchmarks/human_eval/human_eval.py

@@ -92,7 +92,7 @@ class HumanEval(DeepEvalBaseBenchmark):
         self.predictions: Optional[pd.DataFrame] = None
         self.task_scores: Optional[pd.DataFrame] = None
         self.overall_score: Optional[float] = None
-        self.verbose_mode: bool = (False,)
+        self.verbose_mode: bool = verbose_mode
 
     def evaluate(
         self, model: DeepEvalBaseLLM, *args, k: int = 1, **kwargs
@@ -123,6 +123,7 @@ class HumanEval(DeepEvalBaseBenchmark):
                         task.value,
                         golden.input,
                         prediction,
+                        task_correct,
                         golden.expected_output,
                         score,
                     )

deepeval/dataset/dataset.py

@@ -189,17 +189,35 @@ class EvaluationDataset:
         test_case._dataset_alias = self._alias
         test_case._dataset_id = self._id
         if isinstance(test_case, LLMTestCase):
+            if self._conversational_goldens or self._conversational_test_cases:
+                raise TypeError(
+                    "You cannot add 'LLMTestCase' to a multi-turn dataset."
+                )
             test_case._dataset_rank = len(self._llm_test_cases)
             self._llm_test_cases.append(test_case)
         elif isinstance(test_case, ConversationalTestCase):
+            if self._goldens or self._llm_test_cases:
+                raise TypeError(
+                    "You cannot add 'ConversationalTestCase' to a single-turn dataset."
+                )
+            self._multi_turn = True
             test_case._dataset_rank = len(self._conversational_test_cases)
             self._conversational_test_cases.append(test_case)
 
     def add_golden(self, golden: Union[Golden, ConversationalGolden]):
-        if self._multi_turn:
-            self._add_conversational_golden(golden)
-        else:
+        if isinstance(golden, Golden):
+            if self._conversational_goldens or self._conversational_test_cases:
+                raise TypeError(
+                    "You cannot add 'Golden' to a multi-turn dataset."
+                )
             self._add_golden(golden)
+        else:
+            if self._goldens or self._llm_test_cases:
+                raise TypeError(
+                    "You cannot add 'ConversationalGolden' to a single-turn dataset."
+                )
+            self._multi_turn = True
+            self._add_conversational_golden(golden)
 
     def _add_golden(self, golden: Union[Golden, ConversationalGolden]):
         if isinstance(golden, Golden):
@@ -224,16 +242,16 @@ class EvaluationDataset:
         file_path: str,
         input_col_name: str,
         actual_output_col_name: str,
-        expected_output_col_name: Optional[str] = None,
-        context_col_name: Optional[str] = None,
+        expected_output_col_name: Optional[str] = "expected_output",
+        context_col_name: Optional[str] = "context",
         context_col_delimiter: str = ";",
-        retrieval_context_col_name: Optional[str] = None,
+        retrieval_context_col_name: Optional[str] = "retrieval_context",
         retrieval_context_col_delimiter: str = ";",
-        tools_called_col_name: Optional[str] = None,
+        tools_called_col_name: Optional[str] = "tools_called",
         tools_called_col_delimiter: str = ";",
-        expected_tools_col_name: Optional[str] = None,
+        expected_tools_col_name: Optional[str] = "expected_tools",
         expected_tools_col_delimiter: str = ";",
-        additional_metadata_col_name: Optional[str] = None,
+        additional_metadata_col_name: Optional[str] = "additional_metadata",
     ):
         """
         Load test cases from a CSV file.
@@ -379,6 +397,7 @@ class EvaluationDataset:
         retrieval_context_key_name: Optional[str] = None,
         tools_called_key_name: Optional[str] = None,
         expected_tools_key_name: Optional[str] = None,
+        addtional_metadata_key_name: Optional[str] = None,
         encoding_type: str = "utf-8",
     ):
         """
@@ -431,6 +450,7 @@ class EvaluationDataset:
             tools_called = [ToolCall(**tool) for tool in tools_called_data]
             expected_tools_data = json_obj.get(expected_tools_key_name, [])
             expected_tools = [ToolCall(**tool) for tool in expected_tools_data]
+            # additional_metadata = json_obj.get(addtional_metadata_key_name)
 
             self.add_test_case(
                 LLMTestCase(
@@ -441,6 +461,7 @@ class EvaluationDataset:
                     retrieval_context=retrieval_context,
                     tools_called=tools_called,
                     expected_tools=expected_tools,
+                    # additional_metadata=additional_metadata,
                 )
             )
 
@@ -460,8 +481,8 @@ class EvaluationDataset:
         expected_tools_col_delimiter: str = ";",
         comments_key_name: str = "comments",
         name_key_name: str = "name",
-        source_file_col_name: Optional[str] = None,
-        additional_metadata_col_name: Optional[str] = None,
+        source_file_col_name: Optional[str] = "source_file",
+        additional_metadata_col_name: Optional[str] = "additional_metadata",
         scenario_col_name: Optional[str] = "scenario",
         turns_col_name: Optional[str] = "turns",
         expected_outcome_col_name: Optional[str] = "expected_outcome",
@@ -587,6 +608,7 @@ class EvaluationDataset:
                         context=context,
                         comments=comments,
                         name=name,
+                        additional_metadata=additional_metadata,
                     )
                 )
             else:
@@ -645,6 +667,7 @@ class EvaluationDataset:
                 comments = json_obj.get(comments_key_name)
                 name = json_obj.get(name_key_name)
                 parsed_turns = parse_turns(turns) if turns else []
+                additional_metadata = json_obj.get(additional_metadata_key_name)
 
                 self._multi_turn = True
                 self.goldens.append(
@@ -656,6 +679,7 @@ class EvaluationDataset:
                         context=context,
                         comments=comments,
                         name=name,
+                        additional_metadata=additional_metadata,
                     )
                 )
             else:

deepeval/dataset/utils.py

@@ -24,6 +24,7 @@ def convert_test_cases_to_goldens(
             "retrieval_context": test_case.retrieval_context,
             "tools_called": test_case.tools_called,
             "expected_tools": test_case.expected_tools,
+            "additional_metadata": test_case.additional_metadata,
         }
         goldens.append(Golden(**golden))
     return goldens
@@ -70,6 +71,7 @@ def convert_convo_test_cases_to_convo_goldens(
             "expected_outcome": test_case.expected_outcome,
             "user_description": test_case.user_description,
             "context": test_case.context,
+            "additional_metadata": test_case.additional_metadata,
         }
         goldens.append(ConversationalGolden(**golden))
     return goldens

deepeval/evaluate/compare.py

@@ -502,10 +502,14 @@ def wrap_up_experiment(
 
     try:
         api = Api()
-        experiment_request = PostExperimentRequest(testRuns=test_runs, name=name)
+        experiment_request = PostExperimentRequest(
+            testRuns=test_runs, name=name
+        )
 
         try:
-            body = experiment_request.model_dump(by_alias=True, exclude_none=True)
+            body = experiment_request.model_dump(
+                by_alias=True, exclude_none=True
+            )
         except AttributeError:
             body = experiment_request.dict(by_alias=True, exclude_none=True)
         json_str = json.dumps(body, cls=TestRunEncoder)

deepeval/metrics/utils.py

@@ -25,6 +25,7 @@ from deepeval.models import (
     MultimodalOpenAIModel,
     MultimodalGeminiModel,
     MultimodalOllamaModel,
+    MultimodalAzureOpenAIMLLMModel,
     AmazonBedrockModel,
     LiteLLMModel,
     KimiModel,
@@ -514,6 +515,8 @@ def initialize_multimodal_model(
         return MultimodalGeminiModel(), True
     if should_use_ollama_model():
         return MultimodalOllamaModel(), True
+    elif should_use_azure_openai():
+        return MultimodalAzureOpenAIMLLMModel(model_name=model), True
     elif isinstance(model, str) or model is None:
         return MultimodalOpenAIModel(model=model), True
     raise TypeError(

deepeval/models/__init__.py

@@ -21,6 +21,7 @@ from deepeval.models.mlllms import (
     MultimodalOpenAIModel,
     MultimodalOllamaModel,
     MultimodalGeminiModel,
+    MultimodalAzureOpenAIMLLMModel,
 )
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
@@ -48,6 +49,7 @@ __all__ = [
     "MultimodalOpenAIModel",
     "MultimodalOllamaModel",
     "MultimodalGeminiModel",
+    "MultimodalAzureOpenAIMLLMModel",
     "OpenAIEmbeddingModel",
     "AzureOpenAIEmbeddingModel",
     "LocalEmbeddingModel",

deepeval/models/mlllms/__init__.py

@@ -1,3 +1,4 @@
 from .openai_model import MultimodalOpenAIModel
 from .ollama_model import MultimodalOllamaModel
 from .gemini_model import MultimodalGeminiModel
+from .azure_model import MultimodalAzureOpenAIMLLMModel

deepeval/models/mlllms/azure_model.py

@@ -0,0 +1,334 @@
+from openai.types.chat.chat_completion import ChatCompletion
+from openai import AzureOpenAI, AsyncAzureOpenAI
+from typing import Optional, Tuple, Union, Dict, List
+from pydantic import BaseModel
+from io import BytesIO
+import base64
+
+from deepeval.models import DeepEvalBaseMLLM
+from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
+from deepeval.test_case import MLLMImage
+from deepeval.models.llms.openai_model import (
+    structured_outputs_models,
+    json_mode_models,
+    model_pricing,
+)
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
+
+from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import parse_model_name
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_azure = create_retry_decorator(PS.AZURE)
+
+
+class MultimodalAzureOpenAIMLLMModel(DeepEvalBaseMLLM):
+    def __init__(
+        self,
+        deployment_name: Optional[str] = None,
+        model_name: Optional[str] = None,
+        azure_openai_api_key: Optional[str] = None,
+        openai_api_version: Optional[str] = None,
+        azure_endpoint: Optional[str] = None,
+        temperature: float = 0,
+        generation_kwargs: Optional[Dict] = None,
+        **kwargs,
+    ):
+        # fetch Azure deployment parameters
+        model_name = model_name or KEY_FILE_HANDLER.fetch_data(
+            ModelKeyValues.AZURE_MODEL_NAME
+        )
+        self.deployment_name = deployment_name or KEY_FILE_HANDLER.fetch_data(
+            ModelKeyValues.AZURE_DEPLOYMENT_NAME
+        )
+        self.azure_openai_api_key = (
+            azure_openai_api_key
+            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.AZURE_OPENAI_API_KEY)
+        )
+        self.openai_api_version = (
+            openai_api_version
+            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.OPENAI_API_VERSION)
+        )
+        self.azure_endpoint = azure_endpoint or KEY_FILE_HANDLER.fetch_data(
+            ModelKeyValues.AZURE_OPENAI_ENDPOINT
+        )
+        if temperature < 0:
+            raise ValueError("Temperature must be >= 0.")
+        self.temperature = temperature
+
+        # args and kwargs will be passed to the underlying model, in load_model function
+        self.kwargs = kwargs
+        self.generation_kwargs = generation_kwargs or {}
+        super().__init__(parse_model_name(model_name))
+
+    ###############################################
+    # Generate functions
+    ###############################################
+
+    @retry_azure
+    def generate(
+        self,
+        multimodal_input: List[Union[str, MLLMImage]],
+        schema: Optional[BaseModel] = None,
+    ) -> Tuple[Union[str, BaseModel], float]:
+        client = self.load_model(async_mode=False)
+        prompt = self.generate_prompt(multimodal_input)
+
+        if schema:
+            if self.model_name in structured_outputs_models:
+                messages = [{"role": "user", "content": prompt}]
+                completion = client.beta.chat.completions.parse(
+                    model=self.deployment_name,
+                    messages=messages,
+                    response_format=schema,
+                    temperature=self.temperature,
+                )
+                structured_output: BaseModel = completion.choices[
+                    0
+                ].message.parsed
+                cost = self.calculate_cost(
+                    completion.usage.prompt_tokens,
+                    completion.usage.completion_tokens,
+                )
+                return structured_output, cost
+            if self.model_name in json_mode_models:
+                messages = [{"role": "user", "content": prompt}]
+                completion = client.beta.chat.completions.parse(
+                    model=self.deployment_name,
+                    messages=messages,
+                    response_format={"type": "json_object"},
+                    temperature=self.temperature,
+                )
+                json_output = trim_and_load_json(
+                    completion.choices[0].message.content
+                )
+                cost = self.calculate_cost(
+                    completion.usage.prompt_tokens,
+                    completion.usage.completion_tokens,
+                )
+                return schema.model_validate(json_output), cost
+        print("Loading model client:")
+        print(client.base_url)
+        completion = client.chat.completions.create(
+            model=self.deployment_name,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.temperature,
+            **self.generation_kwargs,
+        )
+        output = completion.choices[0].message.content
+        cost = self.calculate_cost(
+            completion.usage.prompt_tokens, completion.usage.completion_tokens
+        )
+        if schema:
+            json_output = trim_and_load_json(output)
+            return schema.model_validate(json_output), cost
+        else:
+            return output, cost
+
+    @retry_azure
+    async def a_generate(
+        self,
+        multimodal_input: List[Union[str, MLLMImage]],
+        schema: Optional[BaseModel] = None,
+    ) -> Tuple[Union[str, BaseModel], float]:
+        client = self.load_model(async_mode=True)
+        prompt = self.generate_prompt(multimodal_input)
+
+        if schema:
+            if self.model_name in structured_outputs_models:
+                messages = [{"role": "user", "content": prompt}]
+                completion = await client.beta.chat.completions.parse(
+                    model=self.deployment_name,
+                    messages=messages,
+                    response_format=schema,
+                    temperature=self.temperature,
+                )
+                structured_output: BaseModel = completion.choices[
+                    0
+                ].message.parsed
+                cost = self.calculate_cost(
+                    completion.usage.prompt_tokens,
+                    completion.usage.completion_tokens,
+                )
+                return structured_output, cost
+            if self.model_name in json_mode_models:
+                messages = [{"role": "user", "content": prompt}]
+                completion = await client.beta.chat.completions.parse(
+                    model=self.deployment_name,
+                    messages=messages,
+                    response_format={"type": "json_object"},
+                    temperature=self.temperature,
+                    **self.generation_kwargs,
+                )
+                json_output = trim_and_load_json(
+                    completion.choices[0].message.content
+                )
+                cost = self.calculate_cost(
+                    completion.usage.prompt_tokens,
+                    completion.usage.completion_tokens,
+                )
+                return schema.model_validate(json_output), cost
+
+        completion = await client.chat.completions.create(
+            model=self.deployment_name,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.temperature,
+            **self.generation_kwargs,
+        )
+        output = completion.choices[0].message.content
+        cost = self.calculate_cost(
+            completion.usage.prompt_tokens,
+            completion.usage.completion_tokens,
+        )
+        if schema:
+            json_output = trim_and_load_json(output)
+            return schema.model_validate(json_output), cost
+        else:
+            return output, cost
+
+    ###############################################
+    # Other generate functions
+    ###############################################
+
+    @retry_azure
+    def generate_raw_response(
+        self,
+        multimodal_input: List[Union[str, MLLMImage]],
+        top_logprobs: int = 5,
+    ) -> Tuple[ChatCompletion, float]:
+        client = self.load_model(async_mode=False)
+        prompt = self.generate_prompt(multimodal_input)
+        messages = [{"role": "user", "content": prompt}]
+
+        # Generate completion
+        completion = client.chat.completions.create(
+            model=self.deployment_name,
+            messages=messages,
+            temperature=self.temperature,
+            logprobs=True,
+            top_logprobs=top_logprobs,
+            **self.generation_kwargs,
+        )
+        # Cost calculation
+        input_tokens = completion.usage.prompt_tokens
+        output_tokens = completion.usage.completion_tokens
+        cost = self.calculate_cost(input_tokens, output_tokens)
+
+        return completion, cost
+
+    @retry_azure
+    async def a_generate_raw_response(
+        self,
+        multimodal_input: List[Union[str, MLLMImage]],
+        top_logprobs: int = 5,
+    ) -> Tuple[ChatCompletion, float]:
+        client = self.load_model(async_mode=True)
+        prompt = self.generate_prompt(multimodal_input)
+        messages = [{"role": "user", "content": prompt}]
+
+        # Generate completion
+        completion = await client.chat.completions.create(
+            model=self.deployment_name,
+            messages=messages,
+            temperature=self.temperature,
+            logprobs=True,
+            top_logprobs=top_logprobs,
+            **self.generation_kwargs,
+        )
+        # Cost calculation
+        input_tokens = completion.usage.prompt_tokens
+        output_tokens = completion.usage.completion_tokens
+        cost = self.calculate_cost(input_tokens, output_tokens)
+
+        return completion, cost
+
+    ###############################################
+    # Utilities
+    ###############################################
+
+    def generate_prompt(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        """Convert multimodal input into the proper message format for Azure OpenAI."""
+        prompt = []
+        for ele in multimodal_input:
+            if isinstance(ele, str):
+                prompt.append({"type": "text", "text": ele})
+            elif isinstance(ele, MLLMImage):
+                if ele.local:
+                    import PIL.Image
+
+                    image = PIL.Image.open(ele.url)
+                    visual_dict = {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{self.encode_pil_image(image)}"
+                        },
+                    }
+                else:
+                    visual_dict = {
+                        "type": "image_url",
+                        "image_url": {"url": ele.url},
+                    }
+                prompt.append(visual_dict)
+        return prompt
+
+    def encode_pil_image(self, pil_image):
+        """Encode a PIL image to base64 string."""
+        image_buffer = BytesIO()
+        if pil_image.mode in ("RGBA", "LA", "P"):
+            pil_image = pil_image.convert("RGB")
+        pil_image.save(image_buffer, format="JPEG")
+        image_bytes = image_buffer.getvalue()
+        base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
+        return base64_encoded_image
+
+    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        pricing = model_pricing.get(self.model_name, model_pricing["gpt-4.1"])
+        input_cost = input_tokens * pricing["input"]
+        output_cost = output_tokens * pricing["output"]
+        return input_cost + output_cost
+
+    ###############################################
+    # Model
+    ###############################################
+
+    def get_model_name(self):
+        return f"Azure OpenAI ({self.model_name})"
+
+    def load_model(self, async_mode: bool = False):
+        if not async_mode:
+            return self._build_client(AzureOpenAI)
+        return self._build_client(AsyncAzureOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'azure' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.AZURE):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self.azure_openai_api_key,
+            api_version=self.openai_api_version,
+            azure_endpoint=self.azure_endpoint,
+            azure_deployment=self.deployment_name,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/synthesizer/config.py

@@ -41,6 +41,15 @@ class StylingConfig:
     expected_output_format: Optional[str] = None
 
 
+@dataclass
+class ConversationalStylingConfig:
+    scenario_context: Optional[str] = None
+    conversational_task: Optional[str] = None
+    participant_roles: Optional[str] = None
+    scenario_format: Optional[str] = None
+    expected_outcome_format: Optional[str] = None
+
+
 @dataclass
 class ContextConstructionConfig:
     embedder: Optional[Union[str, DeepEvalBaseEmbeddingModel]] = None

deepeval/synthesizer/schema.py

@@ -58,3 +58,26 @@ class PromptStyling(BaseModel):
     scenario: str
     task: str
     input_format: str
+
+
+class ConversationalScenario(BaseModel):
+    scenario: str
+
+
+class ConversationalScenarioList(BaseModel):
+    data: List[ConversationalScenario]
+
+
+class RewrittenScenario(BaseModel):
+    rewritten_scenario: str
+
+
+class ScenarioFeedback(BaseModel):
+    score: float
+    feedback: str
+
+
+class ConversationalPromptStyling(BaseModel):
+    scenario_context: str
+    conversational_task: str
+    participant_roles: str