deepeval 3.7.6__py3-none-any.whl → 3.7.8__py3-none-any.whl

deepeval/key_handler.py

@@ -5,7 +5,9 @@ import json
 import logging
 
 from enum import Enum
-from typing import Union
+from functools import lru_cache
+from pydantic import SecretStr
+from typing import get_args, get_origin, Union
 
 from .constants import KEY_FILE, HIDDEN_DIR
 
@@ -13,26 +15,34 @@ from .constants import KEY_FILE, HIDDEN_DIR
 logger = logging.getLogger(__name__)
 
 
-SECRET_KEYS = {
-    # General providers
-    "OPENAI_API_KEY",
-    "ANTHROPIC_API_KEY",
-    # Azure OpenAI
-    "AZURE_OPENAI_API_KEY",
-    # Google / Gemini
-    "GOOGLE_API_KEY",
-    # xAI Grok
-    "GROK_API_KEY",
-    # Moonshot
-    "MOONSHOT_API_KEY",
-    # DeepSeek
-    "DEEPSEEK_API_KEY",
-    # LiteLLM
-    "LITELLM_API_KEY",
-    # Local gateways (if any require keys)
-    "LOCAL_MODEL_API_KEY",
-    "LOCAL_EMBEDDING_API_KEY",
-}
+@lru_cache(maxsize=1)
+def _secret_env_keys() -> frozenset[str]:
+    # Lazy import avoids cycles at import time
+    from deepeval.config.settings import Settings
+
+    secret_keys: set[str] = set()
+    for env_key, field in Settings.model_fields.items():
+        ann = field.annotation
+        if ann is SecretStr:
+            secret_keys.add(env_key)
+            continue
+
+        origin = get_origin(ann)
+        if origin is Union and any(a is SecretStr for a in get_args(ann)):
+            secret_keys.add(env_key)
+
+    return frozenset(secret_keys)
+
+
+def _env_key_for_legacy_enum(key) -> str:
+    # For ModelKeyValues, .name == .value, for KeyValues it's the important one:
+    # KeyValues.API_KEY.name == "API_KEY" (matches Settings), value == "api_key" (legacy json key)
+    return getattr(key, "name", str(key))
+
+
+def _is_secret_key(key) -> bool:
+    return _env_key_for_legacy_enum(key) in _secret_env_keys()
+
 
 _WARNED_SECRET_KEYS = set()
 
@@ -40,7 +50,10 @@ _WARNED_SECRET_KEYS = set()
 class KeyValues(Enum):
     # Confident AI
     API_KEY = "api_key"
+    CONFIDENT_API_KEY = "confident_api_key"
+    CONFIDENT_BASE_URL = "confident_base_url"
     CONFIDENT_REGION = "confident_region"
+
     # Cache
     LAST_TEST_RUN_LINK = "last_test_run_link"
     LAST_TEST_RUN_DATA = "last_test_run_data"
@@ -49,6 +62,24 @@ class KeyValues(Enum):
 class ModelKeyValues(Enum):
     # General
     TEMPERATURE = "TEMPERATURE"
+
+    # Anthropic
+    USE_ANTHROPIC_MODEL = "USE_ANTHROPIC_MODEL"
+    ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY"
+    ANTHROPIC_MODEL_NAME = "ANTHROPIC_MODEL_NAME"
+    ANTHROPIC_COST_PER_INPUT_TOKEN = "ANTHROPIC_COST_PER_INPUT_TOKEN"
+    ANTHROPIC_COST_PER_OUTPUT_TOKEN = "ANTHROPIC_COST_PER_OUTPUT_TOKEN"
+
+    # AWS
+    AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
+    AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
+    # AWS Bedrock
+    USE_AWS_BEDROCK_MODEL = "USE_AWS_BEDROCK_MODEL"
+    AWS_BEDROCK_MODEL_NAME = "AWS_BEDROCK_MODEL_NAME"
+    AWS_BEDROCK_REGION = "AWS_BEDROCK_REGION"
+    AWS_BEDROCK_COST_PER_INPUT_TOKEN = "AWS_BEDROCK_COST_PER_INPUT_TOKEN"
+    AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = "AWS_BEDROCK_COST_PER_OUTPUT_TOKEN"
+
     # Azure Open AI
     AZURE_OPENAI_API_KEY = "AZURE_OPENAI_API_KEY"
     AZURE_OPENAI_ENDPOINT = "AZURE_OPENAI_ENDPOINT"
@@ -57,43 +88,79 @@ class ModelKeyValues(Enum):
     AZURE_MODEL_NAME = "AZURE_MODEL_NAME"
     AZURE_MODEL_VERSION = "AZURE_MODEL_VERSION"
     USE_AZURE_OPENAI = "USE_AZURE_OPENAI"
-    # Local Model
-    LOCAL_MODEL_NAME = "LOCAL_MODEL_NAME"
-    LOCAL_MODEL_BASE_URL = "LOCAL_MODEL_BASE_URL"
-    LOCAL_MODEL_API_KEY = "LOCAL_MODEL_API_KEY"
-    LOCAL_MODEL_FORMAT = "LOCAL_MODEL_FORMAT"
-    USE_LOCAL_MODEL = "USE_LOCAL_MODEL"
+
+    # DeepSeek
+    USE_DEEPSEEK_MODEL = "USE_DEEPSEEK_MODEL"
+    DEEPSEEK_API_KEY = "DEEPSEEK_API_KEY"
+    DEEPSEEK_MODEL_NAME = "DEEPSEEK_MODEL_NAME"
+    DEEPSEEK_COST_PER_INPUT_TOKEN = "DEEPSEEK_COST_PER_INPUT_TOKEN"
+    DEEPSEEK_COST_PER_OUTPUT_TOKEN = "DEEPSEEK_COST_PER_OUTPUT_TOKEN"
+
     # Gemini
     USE_GEMINI_MODEL = "USE_GEMINI_MODEL"
-    GEMINI_MODEL_NAME = "GEMINI_MODEL_NAME"
     GOOGLE_API_KEY = "GOOGLE_API_KEY"
+    GEMINI_MODEL_NAME = "GEMINI_MODEL_NAME"
     GOOGLE_GENAI_USE_VERTEXAI = "GOOGLE_GENAI_USE_VERTEXAI"
     GOOGLE_CLOUD_PROJECT = "GOOGLE_CLOUD_PROJECT"
     GOOGLE_CLOUD_LOCATION = "GOOGLE_CLOUD_LOCATION"
     GOOGLE_SERVICE_ACCOUNT_KEY = "GOOGLE_SERVICE_ACCOUNT_KEY"
+
+    # Grok
+    USE_GROK_MODEL = "USE_GROK_MODEL"
+    GROK_API_KEY = "GROK_API_KEY"
+    GROK_MODEL_NAME = "GROK_MODEL_NAME"
+    GROK_COST_PER_INPUT_TOKEN = "GROK_COST_PER_INPUT_TOKEN"
+    GROK_COST_PER_OUTPUT_TOKEN = "GROK_COST_PER_OUTPUT_TOKEN"
+
     # LiteLLM
     USE_LITELLM = "USE_LITELLM"
-    LITELLM_MODEL_NAME = "LITELLM_MODEL_NAME"
     LITELLM_API_KEY = "LITELLM_API_KEY"
+    LITELLM_MODEL_NAME = "LITELLM_MODEL_NAME"
     LITELLM_API_BASE = "LITELLM_API_BASE"
+    LITELLM_PROXY_API_BASE = "LITELLM_PROXY_API_BASE"
+    LITELLM_PROXY_API_KEY = "LITELLM_PROXY_API_KEY"
+
+    # LM Studio
+    LM_STUDIO_API_KEY = "LM_STUDIO_API_KEY"
+    LM_STUDIO_MODEL_NAME = "LM_STUDIO_MODEL_NAME"
+
+    # Local Model
+    USE_LOCAL_MODEL = "USE_LOCAL_MODEL"
+    LOCAL_MODEL_API_KEY = "LOCAL_MODEL_API_KEY"
+    LOCAL_MODEL_NAME = "LOCAL_MODEL_NAME"
+    LOCAL_MODEL_BASE_URL = "LOCAL_MODEL_BASE_URL"
+    LOCAL_MODEL_FORMAT = "LOCAL_MODEL_FORMAT"
+
+    # Moonshot
+    USE_MOONSHOT_MODEL = "USE_MOONSHOT_MODEL"
+    MOONSHOT_API_KEY = "MOONSHOT_API_KEY"
+    MOONSHOT_MODEL_NAME = "MOONSHOT_MODEL_NAME"
+    MOONSHOT_COST_PER_INPUT_TOKEN = "MOONSHOT_COST_PER_INPUT_TOKEN"
+    MOONSHOT_COST_PER_OUTPUT_TOKEN = "MOONSHOT_COST_PER_OUTPUT_TOKEN"
+
+    # Ollama
+    OLLAMA_MODEL_NAME = "OLLAMA_MODEL_NAME"
+
     # OpenAI
     USE_OPENAI_MODEL = "USE_OPENAI_MODEL"
+    OPENAI_API_KEY = "OPENAI_API_KEY"
     OPENAI_MODEL_NAME = "OPENAI_MODEL_NAME"
     OPENAI_COST_PER_INPUT_TOKEN = "OPENAI_COST_PER_INPUT_TOKEN"
     OPENAI_COST_PER_OUTPUT_TOKEN = "OPENAI_COST_PER_OUTPUT_TOKEN"
-    OPENAI_API_KEY = "OPENAI_API_KEY"
-    # Moonshot
-    USE_MOONSHOT_MODEL = "USE_MOONSHOT_MODEL"
-    MOONSHOT_MODEL_NAME = "MOONSHOT_MODEL_NAME"
-    MOONSHOT_API_KEY = "MOONSHOT_API_KEY"
-    # Grok
-    USE_GROK_MODEL = "USE_GROK_MODEL"
-    GROK_MODEL_NAME = "GROK_MODEL_NAME"
-    GROK_API_KEY = "GROK_API_KEY"
-    # DeepSeek
-    USE_DEEPSEEK_MODEL = "USE_DEEPSEEK_MODEL"
-    DEEPSEEK_MODEL_NAME = "DEEPSEEK_MODEL_NAME"
-    DEEPSEEK_API_KEY = "DEEPSEEK_API_KEY"
+
+    # PortKey
+    USE_PORTKEY_MODEL = "USE_PORTKEY_MODEL"
+    PORTKEY_API_KEY = "PORTKEY_API_KEY"
+    PORTKEY_MODEL_NAME = "PORTKEY_MODEL_NAME"
+    PORTKEY_BASE_URL = "PORTKEY_BASE_URL"
+    PORTKEY_PROVIDER_NAME = "PORTKEY_PROVIDER_NAME"
+
+    # Vertex AI
+    VERTEX_AI_MODEL_NAME = "VERTEX_AI_MODEL_NAME"
+
+    # VLLM
+    VLLM_API_KEY = "VLLM_API_KEY"
+    VLLM_MODEL_NAME = "VLLM_MODEL_NAME"
 
 
 class EmbeddingKeyValues(Enum):
@@ -123,9 +190,11 @@ class KeyFileHandler:
         """Appends or updates data in the hidden file"""
 
         # hard stop on secrets: never write to disk
-        if key.value in SECRET_KEYS:
+        if _is_secret_key(key):
             logger.warning(
-                f"{key} is blacklisted, refusing to persist. Keep your secrets in .env or .env.local instead"
+                "%s is a secret setting, refusing to persist. "
+                "Keep your secrets in .env or .env.local instead.",
+                _env_key_for_legacy_enum(key),
             )
             return
 
@@ -170,16 +239,17 @@ class KeyFileHandler:
         # Deprecation: warn only if we're actually returning a secret
         if (
             value is not None
-            and key.value in SECRET_KEYS
-            and key.value not in _WARNED_SECRET_KEYS
+            and _is_secret_key(key)
+            and _env_key_for_legacy_enum(key) not in _WARNED_SECRET_KEYS
         ):
             logger.warning(
-                f"Reading secret '{key.value}' from legacy {HIDDEN_DIR}/{KEY_FILE}. "
-                "Persisting API keys in plaintext is deprecated. "
-                "Move this to your environment (.env / .env.local). "
-                "This fallback will be removed in a future release."
+                "Reading secret '%s' from legacy %s/%s. Persisting API keys in plaintext is deprecated. "
+                "Move this to your environment (.env / .env.local). This fallback will be removed in a future release.",
+                _env_key_for_legacy_enum(key),
+                HIDDEN_DIR,
+                KEY_FILE,
             )
-            _WARNED_SECRET_KEYS.add(key.value)
+            _WARNED_SECRET_KEYS.add(_env_key_for_legacy_enum(key))
 
         return value
 

deepeval/metrics/base_metric.py

@@ -49,8 +49,10 @@ class BaseMetric:
         return "Base Metric"
 
     def _accrue_cost(self, cost: float) -> None:
-        if self.evaluation_cost is not None:
+        if self.evaluation_cost is not None and cost is not None:
             self.evaluation_cost += cost
+        else:
+            self.evaluation_cost = None
 
 
 class BaseConversationalMetric:
@@ -94,8 +96,10 @@ class BaseConversationalMetric:
         return "Base Conversational Metric"
 
     def _accrue_cost(self, cost: float) -> None:
-        if self.evaluation_cost is not None:
+        if self.evaluation_cost is not None and cost is not None:
             self.evaluation_cost += cost
+        else:
+            self.evaluation_cost = None
 
 
 class BaseArenaMetric:
@@ -129,5 +133,7 @@ class BaseArenaMetric:
         return "Base Arena Metric"
 
     def _accrue_cost(self, cost: float) -> None:
-        if self.evaluation_cost is not None:
+        if self.evaluation_cost is not None and cost is not None:
             self.evaluation_cost += cost
+        else:
+            self.evaluation_cost = None

deepeval/metrics/g_eval/g_eval.py

@@ -110,10 +110,15 @@ class GEval(BaseMetric):
                     _in_component=_in_component,
                     _additional_context=_additional_context,
                 )
+                settings = get_settings()
                 loop.run_until_complete(
                     asyncio.wait_for(
                         coro,
-                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                        timeout=(
+                            None
+                            if settings.DEEPEVAL_DISABLE_TIMEOUTS
+                            else settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+                        ),
                     )
                 )
             else:

deepeval/metrics/indicator.py

@@ -1,10 +1,11 @@
+import asyncio
+import logging
+import sys
+import time
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 from contextlib import contextmanager
-import sys
 from typing import List, Optional, Union
-import time
-import asyncio
 
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics import (
@@ -16,8 +17,8 @@ from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.test_run.cache import CachedTestCase, Cache
 from deepeval.telemetry import capture_metric_type
 from deepeval.utils import update_pbar
+from deepeval.config.settings import get_settings
 
-import logging
 
 logger = logging.getLogger(__name__)
 
@@ -260,6 +261,9 @@ async def safe_a_measure(
             "Timed out/cancelled while evaluating metric. "
             "Increase DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE or set "
             "DEEPEVAL_LOG_STACK_TRACES=1 for full traceback."
+            if not get_settings().DEEPEVAL_DISABLE_TIMEOUTS
+            else "Cancelled while evaluating metric (DeepEval timeouts are disabled; this likely came from upstream orchestration or the provider/network layer). "
+            "Set DEEPEVAL_LOG_STACK_TRACES=1 for full traceback."
         )
         metric.success = False
 

deepeval/metrics/mcp/mcp_task_completion.py

@@ -14,7 +14,7 @@ from deepeval.metrics.utils import (
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.mcp.schema import Task, TaskScore
+from deepeval.metrics.mcp.schema import Task, TaskScore, Reason
 from deepeval.metrics.mcp.template import MCPTaskCompletionTemplate
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.api import metric_data_manager
@@ -171,14 +171,13 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         prompt = MCPTaskCompletionTemplate.generate_final_reason(
             self.score, self.success, reasons
         )
-
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self, task_scores: List[TaskScore]
@@ -194,13 +193,13 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_task_score(self, task: Task) -> TaskScore:
         prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)

deepeval/metrics/mcp/multi_turn_mcp_use_metric.py

@@ -14,7 +14,7 @@ from deepeval.metrics.utils import (
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.mcp.schema import Task, ArgsScore, ToolScore
+from deepeval.metrics.mcp.schema import Task, ArgsScore, ToolScore, Reason
 from deepeval.metrics.mcp.template import MCPTaskCompletionTemplate
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.api import metric_data_manager
@@ -336,13 +336,13 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self,
@@ -363,13 +363,13 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/mcp/schema.py

@@ -20,3 +20,7 @@ class ToolScore(BaseModel):
 class ArgsScore(BaseModel):
     score: float
     reason: str
+
+
+class Reason(BaseModel):
+    reason: str

deepeval/metrics/mcp/template.py

@@ -148,6 +148,13 @@ JSON:
         Context:
         The reasons are from metrics that were used to evaluate an MCP application by determining whether the model accurately completed a task or called toos and resources with the right arguments.
 
+        **
+        IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+        Example JSON:
+        {{
+            "reason": "The score is <score> because <your_reason>."
+        }}
+
         Inputs:
         - final_score: the averaged score across all interactions.
         - success: whether the metric passed or failed
@@ -173,5 +180,5 @@ JSON:
 
         Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
 
-        The final reason:
+        JSON:
         """

deepeval/metrics/prompt_alignment/prompt_alignment.py

@@ -2,7 +2,11 @@ import asyncio
 
 from typing import Optional, List, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+    get_per_task_timeout,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
@@ -19,7 +23,6 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.prompt_alignment.template import PromptAlignmentTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.prompt_alignment import schema as paschema
-from deepeval.config.settings import get_settings
 
 from deepeval.metrics.api import metric_data_manager
 
@@ -86,7 +89,7 @@ class PromptAlignmentMetric(BaseMetric):
                 loop.run_until_complete(
                     asyncio.wait_for(
                         coro,
-                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                        timeout=get_per_task_timeout(),
                     )
                 )
             else:

deepeval/metrics/tool_use/schema.py

@@ -17,3 +17,7 @@ class ToolSelectionScore(BaseModel):
 class ArgumentCorrectnessScore(BaseModel):
     score: float
     reason: str
+
+
+class Reason(BaseModel):
+    reason: str

deepeval/metrics/tool_use/template.py

@@ -161,6 +161,13 @@ class ToolUseTemplate:
             - The key patterns or trends in the sub-reasons (e.g., consistent correct choices, repeated irrelevant tool calls, missed best-fit tools).
             - A clear statement linking the **score** and **threshold** outcome (e.g., “The agent passed because…” or “Failed because…”).
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <score> because <your_reason>."
+            }}
+
             RULES:
             - Focus on *which tools were selected* and *why that selection pattern was or wasn't appropriate*.
             - Mention specific issues or strengths like redundancy, misuse, or perfect matching.
@@ -178,7 +185,7 @@ class ToolUseTemplate:
             Threshold: {threshold}
             Result: {"PASS" if final_score >= threshold else "FAIL"}
 
-            Final Reason:
+            JSON:
             """
         )
 
@@ -199,6 +206,13 @@ class ToolUseTemplate:
             - The dominant strengths or weaknesses from the sub-reasons (e.g., correct parameterization, missing required fields, generic values, or misaligned arguments).
             - Whether the agent met or fell short of the threshold and why.
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <score> because <your_reason>."
+            }}
+
             RULES:
             - Focus strictly on **argument correctness** and **context alignment** — not which tools were chosen.
             - Reference specific argument-level problems or successes where helpful.
@@ -215,6 +229,6 @@ class ToolUseTemplate:
             Threshold: {threshold}
             Result: {"PASS" if final_score >= threshold else "FAIL"}
 
-            Final Reason:
+            JSON:
             """
         )

deepeval/metrics/tool_use/tool_use.py

@@ -23,6 +23,7 @@ from deepeval.metrics.tool_use.schema import (
     ToolSelectionScore,
     UserInputAndTools,
     ArgumentCorrectnessScore,
+    Reason,
 )
 from deepeval.metrics.api import metric_data_manager
 
@@ -356,13 +357,14 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason_for_argument_correctness(
         self,
@@ -376,13 +378,13 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason_for_tool_selection(
         self, tool_use_scores: List[ToolSelectionScore]
@@ -395,13 +397,13 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason_for_argument_correctness(
         self, argument_correctness_scores: List[ArgumentCorrectnessScore]
@@ -414,13 +416,13 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def is_successful(self) -> bool:
         try:

deepeval/metrics/topic_adherence/schema.py

@@ -14,3 +14,7 @@ class QAPairs(BaseModel):
 class RelevancyVerdict(BaseModel):
     verdict: Literal["TP", "TN", "FP", "FN"]
     reason: str
+
+
+class TopicAdherenceReason(BaseModel):
+    reason: str

deepeval/metrics/topic_adherence/template.py

@@ -149,6 +149,13 @@ class TopicAdherenceTemplate:
                 
                 Your task is to go through these reasons and give a single final explaination that clearly explains why this metric has failed or passed.
 
+                **
+                IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+                Example JSON:
+                {{
+                    "reason": "The score is <score> because <your_reason>."
+                }}
+
                 {TopicAdherenceTemplate.multimodal_rules}
 
                 Pass: {success}
@@ -170,6 +177,6 @@ class TopicAdherenceTemplate:
 
                 Output ONLY the reason, DON"T output anything else.
 
-                Reason:
+                JSON:
             """
         )