deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/turn_relevancy/turn_relevancy.py

@@ -11,15 +11,19 @@ from deepeval.metrics.utils import (
     construct_verbose_logs,
     get_turns_in_sliding_window,
     get_unit_interactions,
-    trimAndLoadJson,
     initialize_model,
     convert_turn_to_dict,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, Turn, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.turn_relevancy.schema import *
+from deepeval.metrics.turn_relevancy.schema import (
+    TurnRelevancyVerdict,
+    TurnRelevancyScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -53,7 +57,12 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -108,7 +117,12 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -148,7 +162,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -162,24 +176,19 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         prompt = TurnRelevancyTemplate.generate_reason(
             score=self.score, irrelevancies=irrelevancies
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TurnRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: TurnRelevancyScoreReason = await self.model.a_generate(
-                    prompt, schema=TurnRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
-    def _generate_reason(self) -> str:
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
+
+    def _generate_reason(self) -> Optional[str]:
+        if self.include_reason is False:
+            return None
+
         irrelevancies: List[Dict[str, str]] = []
         for index, verdict in enumerate(self.verdicts):
             if verdict.verdict.strip().lower() == "no":
@@ -190,22 +199,14 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         prompt = TurnRelevancyTemplate.generate_reason(
             score=self.score, irrelevancies=irrelevancies
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=TurnRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: TurnRelevancyScoreReason = self.model.generate(
-                    prompt, schema=TurnRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdict(
         self, turns_sliding_window: List[Turn]
@@ -215,22 +216,14 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                 convert_turn_to_dict(turn) for turn in turns_sliding_window
             ]
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TurnRelevancyVerdict
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: TurnRelevancyVerdict = await self.model.a_generate(
-                    prompt, schema=TurnRelevancyVerdict
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TurnRelevancyVerdict(**data)
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TurnRelevancyVerdict(**data),
+        )
 
     def _generate_verdict(
         self, turns_sliding_window: List[Turn]
@@ -240,20 +233,14 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                 convert_turn_to_dict(turn) for turn in turns_sliding_window
             ]
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=TurnRelevancyVerdict)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: TurnRelevancyVerdict = self.model.generate(
-                    prompt, schema=TurnRelevancyVerdict
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TurnRelevancyVerdict(**data)
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TurnRelevancyVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TurnRelevancyVerdict(**data),
+        )
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -274,7 +261,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         else:
             try:
                 self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/utils.py

@@ -2,7 +2,17 @@ import inspect
 import json
 import re
 import sys
-from typing import Any, Dict, Optional, List, Union, Tuple
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
 
 from deepeval.errors import (
     MissingTestCaseParamsError,
@@ -22,10 +32,19 @@ from deepeval.models import (
     GeminiModel,
     AmazonBedrockModel,
     LiteLLMModel,
+    PortkeyModel,
     KimiModel,
     GrokModel,
     DeepSeekModel,
 )
+from deepeval.models.llms.constants import (
+    OPENAI_MODELS_DATA,
+    GEMINI_MODELS_DATA,
+    OLLAMA_MODELS_DATA,
+    ANTHROPIC_MODELS_DATA,
+    GROK_MODELS_DATA,
+    KIMI_MODELS_DATA,
+)
 from deepeval.key_handler import (
     ModelKeyValues,
     EmbeddingKeyValues,
@@ -34,7 +53,6 @@ from deepeval.key_handler import (
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
     BaseArenaMetric,
 )
 from deepeval.models.base_model import DeepEvalBaseEmbeddingModel
@@ -49,19 +67,20 @@ from deepeval.test_case import (
     TurnParams,
 )
 
-MULTIMODAL_SUPPORTED_MODELS = [
-    GPTModel,
-    GeminiModel,
-    OllamaModel,
-    AzureOpenAIModel,
-]
+MULTIMODAL_SUPPORTED_MODELS = {
+    GPTModel: OPENAI_MODELS_DATA,
+    GeminiModel: GEMINI_MODELS_DATA,
+    OllamaModel: OLLAMA_MODELS_DATA,
+    AzureOpenAIModel: OPENAI_MODELS_DATA,
+    KimiModel: KIMI_MODELS_DATA,
+    AnthropicModel: ANTHROPIC_MODELS_DATA,
+    GrokModel: GROK_MODELS_DATA,
+}
 
 
 def copy_metrics(
-    metrics: List[
-        Union[BaseMetric, BaseConversationalMetric, BaseMultimodalMetric]
-    ],
-) -> List[Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric]]:
+    metrics: List[Union[BaseMetric, BaseConversationalMetric]],
+) -> List[Union[BaseMetric, BaseConversationalMetric]]:
     copied_metrics = []
     for metric in metrics:
         metric_class = type(metric)
@@ -204,13 +223,21 @@ def check_conversational_test_case_params(
 ):
     if multimodal:
         if not model or not model.supports_multimodal():
-            if model and type(model) in MULTIMODAL_SUPPORTED_MODELS:
+            if model and type(model) in MULTIMODAL_SUPPORTED_MODELS.keys():
+                valid_multimodal_models = []
+                for model_name, model_data in MULTIMODAL_SUPPORTED_MODELS.get(
+                    type(model)
+                ).items():
+                    if callable(model_data):
+                        model_data = model_data()
+                    if model_data.supports_multimodal:
+                        valid_multimodal_models.append(model_name)
                 raise ValueError(
-                    f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(model.__class__.valid_multimodal_models)}."
+                    f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(valid_multimodal_models)}."
                 )
             else:
                 raise ValueError(
-                    f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+                    f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS.keys()])}"
                 )
 
     if isinstance(test_case, ConversationalTestCase) is False:
@@ -245,8 +272,49 @@ def check_conversational_test_case_params(
 def check_llm_test_case_params(
     test_case: LLMTestCase,
     test_case_params: List[LLMTestCaseParams],
+    input_image_count: Optional[int],
+    actual_output_image_count: Optional[int],
     metric: Union[BaseMetric, BaseArenaMetric],
+    model: Optional[DeepEvalBaseLLM] = None,
+    multimodal: Optional[bool] = False,
 ):
+    if multimodal:
+        if not model or not model.supports_multimodal():
+            if model and type(model) in MULTIMODAL_SUPPORTED_MODELS.keys():
+                valid_multimodal_models = []
+                for model_name, model_data in MULTIMODAL_SUPPORTED_MODELS.get(
+                    type(model)
+                ).items():
+                    if callable(model_data):
+                        model_data = model_data()
+                    if model_data.supports_multimodal:
+                        valid_multimodal_models.append(model_name)
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(valid_multimodal_models)}."
+                )
+            else:
+                raise ValueError(
+                    f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS.keys()])}"
+                )
+
+        if input_image_count:
+            count = 0
+            for ele in convert_to_multi_modal_array(test_case.input):
+                if isinstance(ele, MLLMImage):
+                    count += 1
+            if count != input_image_count:
+                error_str = f"Can only evaluate test cases with '{input_image_count}' input images using the '{metric.__name__}' metric. `{count}` found."
+                raise ValueError(error_str)
+
+        if actual_output_image_count:
+            count = 0
+            for ele in convert_to_multi_modal_array(test_case.actual_output):
+                if isinstance(ele, MLLMImage):
+                    count += 1
+            if count != actual_output_image_count:
+                error_str = f"Unable to evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
+                raise ValueError(error_str)
+
     if isinstance(test_case, LLMTestCase) is False:
         error_str = f"Unable to evaluate test cases that are not of type 'LLMTestCase' using the non-conversational '{metric.__name__}' metric."
         metric.error = error_str
@@ -276,6 +344,8 @@ def check_arena_test_case_params(
     arena_test_case: ArenaTestCase,
     test_case_params: List[LLMTestCaseParams],
     metric: BaseArenaMetric,
+    model: Optional[DeepEvalBaseLLM] = None,
+    multimodal: Optional[bool] = False,
 ):
     if not isinstance(arena_test_case, ArenaTestCase):
         raise ValueError(
@@ -296,79 +366,8 @@ def check_arena_test_case_params(
             )
 
     for test_case in cases:
-        check_llm_test_case_params(test_case, test_case_params, metric)
-
-
-def check_mllm_test_case_params(
-    test_case: LLMTestCase,
-    test_case_params: List[LLMTestCaseParams],
-    input_image_count: Optional[int],
-    actual_output_image_count: Optional[int],
-    metric: BaseMetric,
-    model: Optional[DeepEvalBaseLLM] = None,
-):
-    if not model or not model.supports_multimodal():
-        if model and type(model) in MULTIMODAL_SUPPORTED_MODELS:
-            raise ValueError(
-                f"The evaluation model {model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {model.__class__.__name__} provider includes {', '.join(model.__class__.valid_multimodal_models)}."
-            )
-        else:
-            raise ValueError(
-                f"The evaluation model {model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
-            )
-
-    if input_image_count:
-        count = 0
-        for ele in convert_to_multi_modal_array(test_case.input):
-            if isinstance(ele, MLLMImage):
-                count += 1
-        if count != input_image_count:
-            error_str = f"Can only evaluate test cases with '{input_image_count}' input images using the '{metric.__name__}' metric. `{count}` found."
-            raise ValueError(error_str)
-
-    if actual_output_image_count:
-        count = 0
-        for ele in convert_to_multi_modal_array(test_case.actual_output):
-            if isinstance(ele, MLLMImage):
-                count += 1
-        if count != actual_output_image_count:
-            error_str = f"Unable to evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
-            raise ValueError(error_str)
-
-    missing_params = []
-    for param in test_case_params:
-        if getattr(test_case, param.value) is None:
-            missing_params.append(f"'{param.value}'")
-
-    if missing_params:
-        if len(missing_params) == 1:
-            missing_params_str = missing_params[0]
-        elif len(missing_params) == 2:
-            missing_params_str = " and ".join(missing_params)
-        else:
-            missing_params_str = (
-                ", ".join(missing_params[:-1]) + ", and " + missing_params[-1]
-            )
-
-        error_str = f"{missing_params_str} cannot be None for the '{metric.__name__}' metric"
-        metric.error = error_str
-        raise MissingTestCaseParamsError(error_str)
-
-
-def check_mllm_test_cases_params(
-    test_cases: List[LLMTestCase],
-    test_case_params: List[LLMTestCaseParams],
-    input_image_count: Optional[int],
-    actual_output_image_count: Optional[int],
-    metric: BaseMetric,
-):
-    for test_case in test_cases:
-        check_mllm_test_case_params(
-            test_case,
-            test_case_params,
-            input_image_count,
-            actual_output_image_count,
-            metric,
+        check_llm_test_case_params(
+            test_case, test_case_params, None, None, metric, model, multimodal
         )
 
 
@@ -398,11 +397,73 @@ def trimAndLoadJson(
         raise Exception(f"An unexpected error occurred: {str(e)}")
 
 
+SchemaType = TypeVar("SchemaType")
+ReturnType = TypeVar("ReturnType")
+
+
+def generate_with_schema_and_extract(
+    metric: Union[BaseMetric, BaseArenaMetric, BaseConversationalMetric],
+    prompt: Any,
+    schema_cls: Type[SchemaType],
+    *,
+    extract_schema: Callable[[SchemaType], ReturnType],
+    extract_json: Callable[[Dict[str, Any]], ReturnType],
+) -> ReturnType:
+    """
+    Synchronous wrapper:
+    - calls model.generate_with_schema(...)
+    - accrues cost if applicable
+    - if schema instance -> extract_schema
+      else parse JSON -> extract_json
+    """
+    if metric.using_native_model:
+        result, cost = metric.model.generate_with_schema(
+            prompt, schema=schema_cls
+        )
+        metric._accrue_cost(cost)
+    else:
+        result = metric.model.generate_with_schema(prompt, schema=schema_cls)
+    if isinstance(result, schema_cls):
+        return extract_schema(result)
+    data = trimAndLoadJson(result, metric)
+    return extract_json(data)
+
+
+async def a_generate_with_schema_and_extract(
+    metric: Union[BaseMetric, BaseArenaMetric, BaseConversationalMetric],
+    prompt: Any,
+    schema_cls: Type[SchemaType],
+    *,
+    extract_schema: Callable[[SchemaType], ReturnType],
+    extract_json: Callable[[Dict[str, Any]], ReturnType],
+) -> ReturnType:
+    if metric.using_native_model:
+        result, cost = await metric.model.a_generate_with_schema(
+            prompt, schema=schema_cls
+        )
+        metric._accrue_cost(cost)
+    else:
+        result = await metric.model.a_generate_with_schema(
+            prompt, schema=schema_cls
+        )
+
+    if isinstance(result, schema_cls):
+        return extract_schema(result)
+
+    data = trimAndLoadJson(result, metric)
+    return extract_json(data)
+
+
 ###############################################
 # Default Model Providers
 ###############################################
 
 
+def should_use_anthropic_model():
+    value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_ANTHROPIC_MODEL)
+    return value.lower() == "yes" if value is not None else False
+
+
 def should_use_azure_openai():
     value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_AZURE_OPENAI)
     return value.lower() == "yes" if value is not None else False
@@ -414,8 +475,8 @@ def should_use_local_model():
 
 
 def should_use_ollama_model():
-    base_url = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_API_KEY)
-    return base_url == "ollama"
+    value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_API_KEY)
+    return value == "ollama"
 
 
 def should_use_gemini_model():
@@ -433,6 +494,11 @@ def should_use_litellm():
     return value.lower() == "yes" if value is not None else False
 
 
+def should_use_portkey():
+    value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_PORTKEY_MODEL)
+    return value.lower() == "yes" if value is not None else False
+
+
 def should_use_deepseek_model():
     value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_DEEPSEEK_MODEL)
     return value.lower() == "yes" if value is not None else False
@@ -471,6 +537,8 @@ def initialize_model(
         return GeminiModel(), True
     if should_use_litellm():
         return LiteLLMModel(), True
+    if should_use_portkey():
+        return PortkeyModel(), True
     if should_use_ollama_model():
         return OllamaModel(), True
     elif should_use_local_model():
@@ -480,9 +548,11 @@ def initialize_model(
     elif should_use_moonshot_model():
         return KimiModel(model=model), True
     elif should_use_grok_model():
-        return GrokModel(model=model), True
+        return GrokModel(), True
     elif should_use_deepseek_model():
         return DeepSeekModel(model=model), True
+    elif should_use_anthropic_model():
+        return AnthropicModel(), True
     elif isinstance(model, str) or model is None:
         return GPTModel(model=model), True
 

deepeval/models/__init__.py

@@ -15,6 +15,7 @@ from deepeval.models.llms import (
     KimiModel,
     GrokModel,
     DeepSeekModel,
+    PortkeyModel,
 )
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
@@ -42,4 +43,5 @@ __all__ = [
     "AzureOpenAIEmbeddingModel",
     "LocalEmbeddingModel",
     "OllamaEmbeddingModel",
+    "PortkeyModel",
 ]

deepeval/models/base_model.py

@@ -1,6 +1,18 @@
 from abc import ABC, abstractmethod
-from typing import Any, Optional, List
+from typing import Any, Optional, List, Union
 from deepeval.models.utils import parse_model_name
+from dataclasses import dataclass
+
+
+@dataclass
+class DeepEvalModelData:
+    supports_log_probs: Optional[bool] = None
+    supports_multimodal: Optional[bool] = None
+    supports_structured_outputs: Optional[bool] = None
+    supports_json: Optional[bool] = None
+    input_price: Optional[float] = None
+    output_price: Optional[float] = None
+    supports_temperature: Optional[bool] = True
 
 
 class DeepEvalBaseModel(ABC):
@@ -66,9 +78,6 @@ class DeepEvalBaseLLM(ABC):
     def get_model_name(self, *args, **kwargs) -> str:
         return self.name
 
-    def supports_multimodal(self) -> bool:
-        return False
-
     def batch_generate(self, *args, **kwargs) -> List[str]:
         """Runs the model to output LLM responses.
 
@@ -79,8 +88,37 @@ class DeepEvalBaseLLM(ABC):
             "batch_generate is not implemented for this model"
         )
 
-    def supports_multimodal(self):
-        return False
+    # Capabilities
+    def supports_log_probs(self) -> Union[bool, None]:
+        return None
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return None
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return None
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return None
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        return None
+
+    def generate_with_schema(self, *args, schema=None, **kwargs):
+        if schema is not None:
+            try:
+                return self.generate(*args, schema=schema, **kwargs)
+            except TypeError:
+                pass  # this means provider doesn't accept schema kwarg
+        return self.generate(*args, **kwargs)
+
+    async def a_generate_with_schema(self, *args, schema=None, **kwargs):
+        if schema is not None:
+            try:
+                return await self.a_generate(*args, schema=schema, **kwargs)
+            except TypeError:
+                pass
+        return await self.a_generate(*args, **kwargs)
 
 
 class DeepEvalBaseEmbeddingModel(ABC):