deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/evaluate/utils.py

@@ -11,7 +11,6 @@ from deepeval.metrics import (
     ArenaGEval,
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -218,9 +217,9 @@ def validate_assert_test_inputs(
         )
 
     if test_case and metrics:
-        if (
-            isinstance(test_case, LLMTestCase) and not test_case.multimodal
-        ) and not all(isinstance(metric, BaseMetric) for metric in metrics):
+        if (isinstance(test_case, LLMTestCase)) and not all(
+            isinstance(metric, BaseMetric) for metric in metrics
+        ):
             raise ValueError(
                 "All 'metrics' for an 'LLMTestCase' must be instances of 'BaseMetric' only."
             )
@@ -230,18 +229,6 @@ def validate_assert_test_inputs(
             raise ValueError(
                 "All 'metrics' for an 'ConversationalTestCase' must be instances of 'BaseConversationalMetric' only."
             )
-        if (
-            isinstance(test_case, LLMTestCase) and test_case.multimodal
-        ) and not all(
-            (
-                isinstance(metric, BaseMultimodalMetric)
-                or isinstance(metric, BaseMetric)
-            )
-            for metric in metrics
-        ):
-            raise ValueError(
-                "All 'metrics' for multi-modal LLMTestCase must be instances of 'BaseMultimodalMetric' only."
-            )
 
     if not ((golden and observed_callback) or (test_case and metrics)):
         raise ValueError(
@@ -259,7 +246,6 @@ def validate_evaluate_inputs(
         Union[
             List[BaseMetric],
             List[BaseConversationalMetric],
-            List[BaseMultimodalMetric],
         ]
     ] = None,
     metric_collection: Optional[str] = None,
@@ -292,10 +278,9 @@ def validate_evaluate_inputs(
     if test_cases and metrics:
         for test_case in test_cases:
             for metric in metrics:
-                if (
-                    isinstance(test_case, LLMTestCase)
-                    and not test_case.multimodal
-                ) and not isinstance(metric, BaseMetric):
+                if (isinstance(test_case, LLMTestCase)) and not isinstance(
+                    metric, BaseMetric
+                ):
                     raise ValueError(
                         f"Metric {metric.__name__} is not a valid metric for LLMTestCase."
                     )
@@ -306,15 +291,6 @@ def validate_evaluate_inputs(
                     raise ValueError(
                         f"Metric {metric.__name__} is not a valid metric for ConversationalTestCase."
                     )
-                if (
-                    isinstance(test_case, LLMTestCase) and test_case.multimodal
-                ) and not (
-                    isinstance(metric, BaseMultimodalMetric)
-                    or isinstance(metric, BaseMetric)
-                ):
-                    raise ValueError(
-                        f"Metric {metric.__name__} is not a valid metric for multi-modal LLMTestCase."
-                    )
 
 
 def print_test_result(test_result: TestResult, display: TestRunResultDisplay):

deepeval/key_handler.py

@@ -5,7 +5,9 @@ import json
 import logging
 
 from enum import Enum
-from typing import Union
+from functools import lru_cache
+from pydantic import SecretStr
+from typing import get_args, get_origin, Union
 
 from .constants import KEY_FILE, HIDDEN_DIR
 
@@ -13,26 +15,34 @@ from .constants import KEY_FILE, HIDDEN_DIR
 logger = logging.getLogger(__name__)
 
 
-SECRET_KEYS = {
-    # General providers
-    "OPENAI_API_KEY",
-    "ANTHROPIC_API_KEY",
-    # Azure OpenAI
-    "AZURE_OPENAI_API_KEY",
-    # Google / Gemini
-    "GOOGLE_API_KEY",
-    # xAI Grok
-    "GROK_API_KEY",
-    # Moonshot
-    "MOONSHOT_API_KEY",
-    # DeepSeek
-    "DEEPSEEK_API_KEY",
-    # LiteLLM
-    "LITELLM_API_KEY",
-    # Local gateways (if any require keys)
-    "LOCAL_MODEL_API_KEY",
-    "LOCAL_EMBEDDING_API_KEY",
-}
+@lru_cache(maxsize=1)
+def _secret_env_keys() -> frozenset[str]:
+    # Lazy import avoids cycles at import time
+    from deepeval.config.settings import Settings
+
+    secret_keys: set[str] = set()
+    for env_key, field in Settings.model_fields.items():
+        ann = field.annotation
+        if ann is SecretStr:
+            secret_keys.add(env_key)
+            continue
+
+        origin = get_origin(ann)
+        if origin is Union and any(a is SecretStr for a in get_args(ann)):
+            secret_keys.add(env_key)
+
+    return frozenset(secret_keys)
+
+
+def _env_key_for_legacy_enum(key) -> str:
+    # For ModelKeyValues, .name == .value, for KeyValues it's the important one:
+    # KeyValues.API_KEY.name == "API_KEY" (matches Settings), value == "api_key" (legacy json key)
+    return getattr(key, "name", str(key))
+
+
+def _is_secret_key(key) -> bool:
+    return _env_key_for_legacy_enum(key) in _secret_env_keys()
+
 
 _WARNED_SECRET_KEYS = set()
 
@@ -40,7 +50,10 @@ _WARNED_SECRET_KEYS = set()
 class KeyValues(Enum):
     # Confident AI
     API_KEY = "api_key"
+    CONFIDENT_API_KEY = "confident_api_key"
+    CONFIDENT_BASE_URL = "confident_base_url"
     CONFIDENT_REGION = "confident_region"
+
     # Cache
     LAST_TEST_RUN_LINK = "last_test_run_link"
     LAST_TEST_RUN_DATA = "last_test_run_data"
@@ -49,6 +62,24 @@ class KeyValues(Enum):
 class ModelKeyValues(Enum):
     # General
     TEMPERATURE = "TEMPERATURE"
+
+    # Anthropic
+    USE_ANTHROPIC_MODEL = "USE_ANTHROPIC_MODEL"
+    ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY"
+    ANTHROPIC_MODEL_NAME = "ANTHROPIC_MODEL_NAME"
+    ANTHROPIC_COST_PER_INPUT_TOKEN = "ANTHROPIC_COST_PER_INPUT_TOKEN"
+    ANTHROPIC_COST_PER_OUTPUT_TOKEN = "ANTHROPIC_COST_PER_OUTPUT_TOKEN"
+
+    # AWS
+    AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
+    AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
+    # AWS Bedrock
+    USE_AWS_BEDROCK_MODEL = "USE_AWS_BEDROCK_MODEL"
+    AWS_BEDROCK_MODEL_NAME = "AWS_BEDROCK_MODEL_NAME"
+    AWS_BEDROCK_REGION = "AWS_BEDROCK_REGION"
+    AWS_BEDROCK_COST_PER_INPUT_TOKEN = "AWS_BEDROCK_COST_PER_INPUT_TOKEN"
+    AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = "AWS_BEDROCK_COST_PER_OUTPUT_TOKEN"
+
     # Azure Open AI
     AZURE_OPENAI_API_KEY = "AZURE_OPENAI_API_KEY"
     AZURE_OPENAI_ENDPOINT = "AZURE_OPENAI_ENDPOINT"
@@ -57,49 +88,88 @@ class ModelKeyValues(Enum):
     AZURE_MODEL_NAME = "AZURE_MODEL_NAME"
     AZURE_MODEL_VERSION = "AZURE_MODEL_VERSION"
     USE_AZURE_OPENAI = "USE_AZURE_OPENAI"
-    # Local Model
-    LOCAL_MODEL_NAME = "LOCAL_MODEL_NAME"
-    LOCAL_MODEL_BASE_URL = "LOCAL_MODEL_BASE_URL"
-    LOCAL_MODEL_API_KEY = "LOCAL_MODEL_API_KEY"
-    LOCAL_MODEL_FORMAT = "LOCAL_MODEL_FORMAT"
-    USE_LOCAL_MODEL = "USE_LOCAL_MODEL"
+
+    # DeepSeek
+    USE_DEEPSEEK_MODEL = "USE_DEEPSEEK_MODEL"
+    DEEPSEEK_API_KEY = "DEEPSEEK_API_KEY"
+    DEEPSEEK_MODEL_NAME = "DEEPSEEK_MODEL_NAME"
+    DEEPSEEK_COST_PER_INPUT_TOKEN = "DEEPSEEK_COST_PER_INPUT_TOKEN"
+    DEEPSEEK_COST_PER_OUTPUT_TOKEN = "DEEPSEEK_COST_PER_OUTPUT_TOKEN"
+
     # Gemini
     USE_GEMINI_MODEL = "USE_GEMINI_MODEL"
-    GEMINI_MODEL_NAME = "GEMINI_MODEL_NAME"
     GOOGLE_API_KEY = "GOOGLE_API_KEY"
+    GEMINI_MODEL_NAME = "GEMINI_MODEL_NAME"
     GOOGLE_GENAI_USE_VERTEXAI = "GOOGLE_GENAI_USE_VERTEXAI"
     GOOGLE_CLOUD_PROJECT = "GOOGLE_CLOUD_PROJECT"
     GOOGLE_CLOUD_LOCATION = "GOOGLE_CLOUD_LOCATION"
     GOOGLE_SERVICE_ACCOUNT_KEY = "GOOGLE_SERVICE_ACCOUNT_KEY"
+
+    # Grok
+    USE_GROK_MODEL = "USE_GROK_MODEL"
+    GROK_API_KEY = "GROK_API_KEY"
+    GROK_MODEL_NAME = "GROK_MODEL_NAME"
+    GROK_COST_PER_INPUT_TOKEN = "GROK_COST_PER_INPUT_TOKEN"
+    GROK_COST_PER_OUTPUT_TOKEN = "GROK_COST_PER_OUTPUT_TOKEN"
+
     # LiteLLM
     USE_LITELLM = "USE_LITELLM"
-    LITELLM_MODEL_NAME = "LITELLM_MODEL_NAME"
     LITELLM_API_KEY = "LITELLM_API_KEY"
+    LITELLM_MODEL_NAME = "LITELLM_MODEL_NAME"
     LITELLM_API_BASE = "LITELLM_API_BASE"
+    LITELLM_PROXY_API_BASE = "LITELLM_PROXY_API_BASE"
+    LITELLM_PROXY_API_KEY = "LITELLM_PROXY_API_KEY"
+
+    # LM Studio
+    LM_STUDIO_API_KEY = "LM_STUDIO_API_KEY"
+    LM_STUDIO_MODEL_NAME = "LM_STUDIO_MODEL_NAME"
+
+    # Local Model
+    USE_LOCAL_MODEL = "USE_LOCAL_MODEL"
+    LOCAL_MODEL_API_KEY = "LOCAL_MODEL_API_KEY"
+    LOCAL_MODEL_NAME = "LOCAL_MODEL_NAME"
+    LOCAL_MODEL_BASE_URL = "LOCAL_MODEL_BASE_URL"
+    LOCAL_MODEL_FORMAT = "LOCAL_MODEL_FORMAT"
+
+    # Moonshot
+    USE_MOONSHOT_MODEL = "USE_MOONSHOT_MODEL"
+    MOONSHOT_API_KEY = "MOONSHOT_API_KEY"
+    MOONSHOT_MODEL_NAME = "MOONSHOT_MODEL_NAME"
+    MOONSHOT_COST_PER_INPUT_TOKEN = "MOONSHOT_COST_PER_INPUT_TOKEN"
+    MOONSHOT_COST_PER_OUTPUT_TOKEN = "MOONSHOT_COST_PER_OUTPUT_TOKEN"
+
+    # Ollama
+    OLLAMA_MODEL_NAME = "OLLAMA_MODEL_NAME"
+
     # OpenAI
     USE_OPENAI_MODEL = "USE_OPENAI_MODEL"
+    OPENAI_API_KEY = "OPENAI_API_KEY"
     OPENAI_MODEL_NAME = "OPENAI_MODEL_NAME"
     OPENAI_COST_PER_INPUT_TOKEN = "OPENAI_COST_PER_INPUT_TOKEN"
     OPENAI_COST_PER_OUTPUT_TOKEN = "OPENAI_COST_PER_OUTPUT_TOKEN"
-    OPENAI_API_KEY = "OPENAI_API_KEY"
-    # Moonshot
-    USE_MOONSHOT_MODEL = "USE_MOONSHOT_MODEL"
-    MOONSHOT_MODEL_NAME = "MOONSHOT_MODEL_NAME"
-    MOONSHOT_API_KEY = "MOONSHOT_API_KEY"
-    # Grok
-    USE_GROK_MODEL = "USE_GROK_MODEL"
-    GROK_MODEL_NAME = "GROK_MODEL_NAME"
-    GROK_API_KEY = "GROK_API_KEY"
-    # DeepSeek
-    USE_DEEPSEEK_MODEL = "USE_DEEPSEEK_MODEL"
-    DEEPSEEK_MODEL_NAME = "DEEPSEEK_MODEL_NAME"
-    DEEPSEEK_API_KEY = "DEEPSEEK_API_KEY"
+
+    # PortKey
+    USE_PORTKEY_MODEL = "USE_PORTKEY_MODEL"
+    PORTKEY_API_KEY = "PORTKEY_API_KEY"
+    PORTKEY_MODEL_NAME = "PORTKEY_MODEL_NAME"
+    PORTKEY_BASE_URL = "PORTKEY_BASE_URL"
+    PORTKEY_PROVIDER_NAME = "PORTKEY_PROVIDER_NAME"
+
+    # Vertex AI
+    VERTEX_AI_MODEL_NAME = "VERTEX_AI_MODEL_NAME"
+
+    # VLLM
+    VLLM_API_KEY = "VLLM_API_KEY"
+    VLLM_MODEL_NAME = "VLLM_MODEL_NAME"
 
 
 class EmbeddingKeyValues(Enum):
     # Azure OpenAI
     USE_AZURE_OPENAI_EMBEDDING = "USE_AZURE_OPENAI_EMBEDDING"
+    # Azure OpenAI
+    AZURE_EMBEDDING_MODEL_NAME = "AZURE_EMBEDDING_MODEL_NAME"
     AZURE_EMBEDDING_DEPLOYMENT_NAME = "AZURE_EMBEDDING_DEPLOYMENT_NAME"
+
     # Local
     USE_LOCAL_EMBEDDINGS = "USE_LOCAL_EMBEDDINGS"
     LOCAL_EMBEDDING_MODEL_NAME = "LOCAL_EMBEDDING_MODEL_NAME"
@@ -120,9 +190,11 @@ class KeyFileHandler:
         """Appends or updates data in the hidden file"""
 
         # hard stop on secrets: never write to disk
-        if key.value in SECRET_KEYS:
+        if _is_secret_key(key):
             logger.warning(
-                f"{key} is blacklisted, refusing to persist. Keep your secrets in .env or .env.local instead"
+                "%s is a secret setting, refusing to persist. "
+                "Keep your secrets in .env or .env.local instead.",
+                _env_key_for_legacy_enum(key),
             )
             return
 
@@ -167,16 +239,17 @@ class KeyFileHandler:
         # Deprecation: warn only if we're actually returning a secret
         if (
             value is not None
-            and key.value in SECRET_KEYS
-            and key.value not in _WARNED_SECRET_KEYS
+            and _is_secret_key(key)
+            and _env_key_for_legacy_enum(key) not in _WARNED_SECRET_KEYS
         ):
             logger.warning(
-                f"Reading secret '{key.value}' from legacy {HIDDEN_DIR}/{KEY_FILE}. "
-                "Persisting API keys in plaintext is deprecated. "
-                "Move this to your environment (.env / .env.local). "
-                "This fallback will be removed in a future release."
+                "Reading secret '%s' from legacy %s/%s. Persisting API keys in plaintext is deprecated. "
+                "Move this to your environment (.env / .env.local). This fallback will be removed in a future release.",
+                _env_key_for_legacy_enum(key),
+                HIDDEN_DIR,
+                KEY_FILE,
             )
-            _WARNED_SECRET_KEYS.add(key.value)
+            _WARNED_SECRET_KEYS.add(_env_key_for_legacy_enum(key))
 
         return value
 

deepeval/metrics/__init__.py

@@ -1,7 +1,6 @@
 from .base_metric import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
     BaseArenaMetric,
 )
 
@@ -65,7 +64,6 @@ from .multimodal_metrics import (
     ImageCoherenceMetric,
     ImageHelpfulnessMetric,
     ImageReferenceMetric,
-    MultimodalGEval,
 )
 
 
@@ -73,7 +71,6 @@ __all__ = [
     # Base classes
     "BaseMetric",
     "BaseConversationalMetric",
-    "BaseMultimodalMetric",
     "BaseArenaMetric",
     # Non-LLM metrics
     "ExactMatchMetric",
@@ -133,5 +130,4 @@ __all__ = [
     "ImageCoherenceMetric",
     "ImageHelpfulnessMetric",
     "ImageReferenceMetric",
-    "MultimodalGEval",
 ]

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -6,17 +6,22 @@ from deepeval.utils import (
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
-    check_mllm_test_case_params,
     initialize_model,
+    generate_with_schema_and_extract,
+    a_generate_with_schema_and_extract,
 )
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
 from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.answer_relevancy.template import AnswerRelevancyTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.answer_relevancy.schema import *
+from deepeval.metrics.answer_relevancy.schema import (
+    Statements,
+    AnswerRelevancyVerdict,
+    Verdicts,
+    AnswerRelevancyScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -55,13 +60,15 @@ class AnswerRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        multimodal = test_case.multimodal
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -82,13 +89,13 @@ class AnswerRelevancyMetric(BaseMetric):
                 actual_output = test_case.actual_output
 
                 self.statements: List[str] = self._generate_statements(
-                    actual_output, multimodal
+                    actual_output, test_case.multimodal
                 )
                 self.verdicts: List[AnswerRelevancyVerdict] = (
-                    self._generate_verdicts(input, multimodal)
+                    self._generate_verdicts(input, test_case.multimodal)
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(input, multimodal)
+                self.reason = self._generate_reason(input, test_case.multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -113,13 +120,15 @@ class AnswerRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        multimodal = test_case.multimodal
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -132,13 +141,15 @@ class AnswerRelevancyMetric(BaseMetric):
             actual_output = test_case.actual_output
 
             self.statements: List[str] = await self._a_generate_statements(
-                actual_output, multimodal
+                actual_output, test_case.multimodal
             )
             self.verdicts: List[AnswerRelevancyVerdict] = (
-                await self._a_generate_verdicts(input, multimodal)
+                await self._a_generate_verdicts(input, test_case.multimodal)
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(input, multimodal)
+            self.reason = await self._a_generate_reason(
+                input, test_case.multimodal
+            )
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -170,22 +181,13 @@ class AnswerRelevancyMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=AnswerRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: AnswerRelevancyScoreReason = await self.model.a_generate(
-                    prompt=prompt, schema=AnswerRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AnswerRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
@@ -203,22 +205,13 @@ class AnswerRelevancyMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=AnswerRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: AnswerRelevancyScoreReason = self.model.generate(
-                    prompt=prompt, schema=AnswerRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AnswerRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, input: str, multimodal: bool
@@ -230,22 +223,15 @@ class AnswerRelevancyMetric(BaseMetric):
             input=input, statements=self.statements, multimodal=multimodal
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    AnswerRelevancyVerdict(**item) for item in data["verdicts"]
-                ]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                AnswerRelevancyVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(
         self, input: str, multimodal: bool
@@ -257,22 +243,17 @@ class AnswerRelevancyMetric(BaseMetric):
             input=input, statements=self.statements, multimodal=multimodal
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    AnswerRelevancyVerdict(**item) for item in data["verdicts"]
-                ]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                AnswerRelevancyVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
-    async def _a_generate_statements(
+    def _generate_statements(
         self,
         actual_output: str,
         multimodal: bool,
@@ -280,31 +261,18 @@ class AnswerRelevancyMetric(BaseMetric):
         prompt = self.evaluation_template.generate_statements(
             actual_output=actual_output, multimodal=multimodal
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Statements)
-            self.evaluation_cost += cost
-            statements: List[str] = res.statements + [
-                ele for ele in actual_output if isinstance(ele, MLLMImage)
-            ]
-            return statements
-        else:
-            try:
-                res: Statements = await self.model.a_generate(
-                    prompt, schema=Statements
-                )
-                statements: List[str] = res.statements + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                statements = data["statements"] + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
 
-    def _generate_statements(
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Statements,
+            extract_schema=lambda s: s.statements
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+            extract_json=lambda d: d["statements"]
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+        )
+
+    async def _a_generate_statements(
         self,
         actual_output: str,
         multimodal: bool,
@@ -312,27 +280,16 @@ class AnswerRelevancyMetric(BaseMetric):
         prompt = self.evaluation_template.generate_statements(
             actual_output=actual_output, multimodal=multimodal
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Statements)
-            self.evaluation_cost += cost
-            statements = res.statements + [
-                ele for ele in actual_output if isinstance(ele, MLLMImage)
-            ]
-            return statements
-        else:
-            try:
-                res: Statements = self.model.generate(prompt, schema=Statements)
-                statements = res.statements + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                statements = data["statements"] + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Statements,
+            extract_schema=lambda s: s.statements
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+            extract_json=lambda d: d["statements"]
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+        )
 
     def _calculate_score(self):
         number_of_verdicts = len(self.verdicts)
@@ -353,7 +310,7 @@ class AnswerRelevancyMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success