deepeval 3.7.3__tar.gz → 3.7.4__tar.gz

{deepeval-3.7.3 → deepeval-3.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.7.3
+Version: 3.7.4
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.7.4/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.7.4"

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/cli/test.py

@@ -160,7 +160,7 @@ def run(
         pytest_args.extend(["--identifier", identifier])
 
     # Add the deepeval plugin file to pytest arguments
-    pytest_args.extend(["-p", "plugins"])
+    pytest_args.extend(["-p", "deepeval"])
     # Append the extra arguments collected by allow_extra_args=True
     # Pytest will raise its own error if the arguments are invalid (error:
     if ctx.args:

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/config/settings.py

@@ -49,6 +49,8 @@ _DEPRECATED_TO_OVERRIDE = {
     "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS": "DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE",
     "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS": "DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE",
 }
+# Track which secrets we've warned about when loading from the legacy keyfile
+_LEGACY_KEYFILE_SECRET_WARNED: set[str] = set()
 
 
 def _find_legacy_enum(env_key: str):
@@ -88,6 +90,82 @@ def _is_secret_key(settings: "Settings", env_key: str) -> bool:
     return False
 
 
+def _merge_legacy_keyfile_into_env() -> None:
+    """
+    Backwards compatibility: merge values from the legacy .deepeval/.deepeval
+    JSON keystore into os.environ for known Settings fields, without
+    overwriting existing process env vars.
+
+    This runs before we compute the Settings env fingerprint so that Pydantic
+    can see these values on first construction.
+
+    Precedence: process env -> dotenv -> legacy json
+    """
+    # if somebody really wants to skip this behavior
+    if parse_bool(os.getenv("DEEPEVAL_DISABLE_LEGACY_KEYFILE"), default=False):
+        return
+
+    from deepeval.constants import HIDDEN_DIR, KEY_FILE
+    from deepeval.key_handler import (
+        KeyValues,
+        ModelKeyValues,
+        EmbeddingKeyValues,
+        SECRET_KEYS,
+    )
+
+    key_path = Path(HIDDEN_DIR) / KEY_FILE
+
+    try:
+        with key_path.open("r", encoding="utf-8") as f:
+            try:
+                data = json.load(f)
+            except json.JSONDecodeError:
+                # Corrupted file -> ignore, same as KeyFileHandler
+                return
+    except FileNotFoundError:
+        # No legacy store -> nothing to merge
+        return
+
+    if not isinstance(data, dict):
+        return
+
+    # Map JSON keys (enum .value) -> env keys (enum .name)
+    mapping: Dict[str, str] = {}
+    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):
+        for member in enum:
+            mapping[member.value] = member.name
+
+    for json_key, raw in data.items():
+        env_key = mapping.get(json_key)
+        if not env_key:
+            continue
+
+        # Process env always wins
+        if env_key in os.environ:
+            continue
+        if raw is None:
+            continue
+
+        # Mirror the legacy warning semantics for secrets, but only once per key
+        if (
+            json_key in SECRET_KEYS
+            and json_key not in _LEGACY_KEYFILE_SECRET_WARNED
+        ):
+            logger.warning(
+                "Reading secret '%s' from legacy %s/%s. "
+                "Persisting API keys in plaintext is deprecated. "
+                "Move this to your environment (.env / .env.local). "
+                "This fallback will be removed in a future release.",
+                json_key,
+                HIDDEN_DIR,
+                KEY_FILE,
+            )
+            _LEGACY_KEYFILE_SECRET_WARNED.add(json_key)
+
+        # Let Settings validators coerce types; we just inject the raw string
+        os.environ[env_key] = str(raw)
+
+
 def _read_env_file(path: Path) -> Dict[str, str]:
     if not path.exists():
         return {}
@@ -258,6 +336,7 @@ class Settings(BaseSettings):
     GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None
     GOOGLE_CLOUD_PROJECT: Optional[str] = None
     GOOGLE_CLOUD_LOCATION: Optional[str] = None
+    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[str] = None
     # Grok
     USE_GROK_MODEL: Optional[bool] = None
     GROK_API_KEY: Optional[SecretStr] = None
@@ -291,6 +370,12 @@ class Settings(BaseSettings):
     OPENAI_MODEL_NAME: Optional[str] = None
     OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None
     OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None
+    # PortKey
+    USE_PORTKEY_MODEL: Optional[bool] = None
+    PORTKEY_API_KEY: Optional[SecretStr] = None
+    PORTKEY_MODEL_NAME: Optional[str] = None
+    PORTKEY_BASE_URL: Optional[AnyUrl] = None
+    PORTKEY_PROVIDER_NAME: Optional[str] = None
     # Vertex AI
     VERTEX_AI_MODEL_NAME: Optional[str] = None
     # VLLM
@@ -516,29 +601,30 @@ class Settings(BaseSettings):
         "CONFIDENT_OPEN_BROWSER",
         "CONFIDENT_TRACE_FLUSH",
         "CONFIDENT_TRACE_VERBOSE",
+        "CUDA_LAUNCH_BLOCKING",
+        "DEEPEVAL_VERBOSE_MODE",
+        "DEEPEVAL_GRPC_LOGGING",
+        "DEEPEVAL_DISABLE_DOTENV",
+        "DEEPEVAL_TELEMETRY_OPT_OUT",
+        "DEEPEVAL_UPDATE_WARNING_OPT_IN",
+        "ENABLE_DEEPEVAL_CACHE",
+        "ERROR_REPORTING",
+        "GOOGLE_GENAI_USE_VERTEXAI",
+        "IGNORE_DEEPEVAL_ERRORS",
+        "SKIP_DEEPEVAL_MISSING_PARAMS",
+        "TOKENIZERS_PARALLELISM",
+        "TRANSFORMERS_NO_ADVISORY_WARNINGS",
         "USE_OPENAI_MODEL",
         "USE_AZURE_OPENAI",
         "USE_LOCAL_MODEL",
         "USE_GEMINI_MODEL",
-        "GOOGLE_GENAI_USE_VERTEXAI",
         "USE_MOONSHOT_MODEL",
         "USE_GROK_MODEL",
         "USE_DEEPSEEK_MODEL",
         "USE_LITELLM",
         "USE_AZURE_OPENAI_EMBEDDING",
         "USE_LOCAL_EMBEDDINGS",
-        "DEEPEVAL_GRPC_LOGGING",
-        "DEEPEVAL_DISABLE_DOTENV",
-        "DEEPEVAL_TELEMETRY_OPT_OUT",
-        "DEEPEVAL_UPDATE_WARNING_OPT_IN",
-        "TOKENIZERS_PARALLELISM",
-        "TRANSFORMERS_NO_ADVISORY_WARNINGS",
-        "CUDA_LAUNCH_BLOCKING",
-        "ERROR_REPORTING",
-        "IGNORE_DEEPEVAL_ERRORS",
-        "SKIP_DEEPEVAL_MISSING_PARAMS",
-        "DEEPEVAL_VERBOSE_MODE",
-        "ENABLE_DEEPEVAL_CACHE",
+        "USE_PORTKEY_MODEL",
         mode="before",
     )
     @classmethod
@@ -1008,6 +1094,9 @@ _settings_lock = threading.RLock()
 
 
 def _calc_env_fingerprint() -> str:
+    # Pull legacy .deepeval JSON-based settings into the process env before hashing
+    _merge_legacy_keyfile_into_env()
+
     env = os.environ.copy()
     # must hash in a stable order.
     keys = sorted(

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/evaluate/configs.py

@@ -7,7 +7,7 @@ from deepeval.test_run.test_run import TestRunResultDisplay
 @dataclass
 class AsyncConfig:
     run_async: bool = True
-    throttle_value: int = 0
+    throttle_value: float = 0
     max_concurrent: int = 20
 
     def __post_init__(self):

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/evaluate/execute.py

@@ -718,6 +718,8 @@ async def a_execute_test_cases(
                     "Gather timed out after %.1fs. Some metrics may be marked as timed out.",
                     _gather_timeout(),
                 )
+                if not error_config.ignore_errors:
+                    raise
 
     else:
         for test_case in test_cases:
@@ -803,7 +805,8 @@ async def a_execute_test_cases(
                 if not t.done():
                     t.cancel()
             await asyncio.gather(*tasks, return_exceptions=True)
-            raise
+            if not error_config.ignore_errors:
+                raise
 
     return test_results
 

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/answer_relevancy/template.py

@@ -49,12 +49,12 @@ Expected JSON format:
             "verdict": "yes"
         }},
         {{
-            "verdict": "no",
-            "reason": <explanation_for_irrelevance>
+            "reason": <explanation_for_irrelevance>,
+            "verdict": "no"
         }},
         {{
-            "verdict": "idk",
-            "reason": <explanation_for_ambiguity>
+            "reason": <explanation_for_ambiguity>,
+            "verdict": "idk"
         }}
     ]  
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/argument_correctness/template.py

@@ -70,8 +70,8 @@ class ArgumentCorrectnessTemplate:
                         "verdict": "yes"
                     }},
                     {{
-                        "verdict": "no",
-                        "reason": "Recommending romantic Parisian comedies does not help find the highest temperature in 2023."
+                        "reason": "Recommending romantic Parisian comedies does not help find the highest temperature in 2023.",
+                        "verdict": "no"
                     }}
                 ]  
             }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/bias/template.py

@@ -64,15 +64,15 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "The opinion "Government meddling in healthcare bloats costs and quashes innovation" reveals a political bias, emphasizing negative views on government involvement."
+            "reason": "The opinion "Government meddling in healthcare bloats costs and quashes innovation" reveals a political bias, emphasizing negative views on government involvement.",
+            "verdict": "yes"
         }},
         {{
             "verdict": "no"
         }},
         {{
             "verdict": "no"
-        }},
+        }}
     ]  
 }}
 

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/contextual_precision/template.py

@@ -19,16 +19,16 @@ Example:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'"
+            "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'",
+            "verdict": "yes"
         }},
         {{
-            "verdict": "yes",
-            "reason": "The text verifies that the prize was indeed won in 1968."
+            "reason": "The text verifies that the prize was indeed won in 1968.",
+            "verdict": "yes"
         }},
         {{
-            "verdict": "no",
-            "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize."
+            "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize.",
+            "verdict": "no"
         }}
     ]  
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/contextual_recall/template.py

@@ -55,8 +55,8 @@ IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' k
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "..."
+            "reason": "...",
+            "verdict": "yes"
         }},
         ...
     ]  

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/contextual_relevancy/template.py

@@ -55,13 +55,13 @@ Example:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
             "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
+            "verdict": "yes"
         }},
         {{
-            "verdict": "no",
             "statement": "There was a cat.",
-            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements."
+            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
+            "verdict": "no"
         }}
     ]
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/conversation_completeness/template.py

@@ -86,8 +86,8 @@ User wants to tell the assistant something.
 
 Example JSON:
 {{
-    "verdict": "no",
-    "reason": "The user wanted to tell the assistant something but the LLM not only refused to answer but replied 'Oh ok, in that case should you need anything just let me know!', which is completely irrelevant and doesn't satisfy the user at all. "
+    "reason": "The user wanted to tell the assistant something but the LLM not only refused to answer but replied 'Oh ok, in that case should you need anything just let me know!', which is completely irrelevant and doesn't satisfy the user at all.",
+    "verdict": "no"
 }}
 ===== END OF EXAMPLE ======
 

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/conversational_dag/templates.py

@@ -77,8 +77,8 @@ class ConversationalBinaryJudgementTemplate:
 
                 Example:
                 {{
-                "verdict": true,
-                "reason": "The assistant provided a clear and direct answer in response to every user query."
+                "reason": "The assistant provided a clear and direct answer in response to every user query.",
+                "verdict": true
                 }}
                 **
                 JSON:
@@ -108,8 +108,8 @@ class ConversationalNonBinaryJudgementTemplate:
 
                 Example:
                 {{
-                "verdict": "{options[1]}",
-                "reason": "The assistant partially addressed the user’s issue but missed clarifying their follow-up question."
+                "reason": "The assistant partially addressed the user's issue but missed clarifying their follow-up question.",
+                "verdict": "{options[1]}"
                 }}
                 **
                 JSON:

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/conversational_g_eval/template.py

@@ -70,7 +70,8 @@ JSON:
     ---
     Example JSON:
     {{
-    "score": 0,
-    "reason": "Your concise and informative reason here."
+        "reason": "Your concise and informative reason here.",
+        "score": 0
     }}
-    """
+
+    JSON:"""

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/dag/templates.py

@@ -63,8 +63,8 @@ class BinaryJudgementTemplate:
 IMPORTANT: Please make sure to only return a json with two keys: `verdict` (True or False), and the 'reason' key providing the reason. The verdict must be a boolean only, either True or False.
 Example JSON:
 {{
-    "verdict": True,
-    "reason": "..."
+    "reason": "...",
+    "verdict": True
 }}
 **
 
@@ -85,8 +85,8 @@ class NonBinaryJudgementTemplate:
 IMPORTANT: Please make sure to only return a json with two keys: 'verdict' {options} and 'reason' providing the reason.
 Example JSON:
 {{
-    "verdict": {options},
-    "reason": "..."
+    "reason": "...",
+    "verdict": {options}
 }}
 **
 

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/faithfulness/template.py

@@ -83,12 +83,12 @@ Expected JSON format:
             "verdict": "yes"
         }},
         {{
-            "verdict": "no",
-            "reason": <explanation_for_contradiction>
+            "reason": <explanation_for_contradiction>,
+            "verdict": "no"
         }},
         {{
-            "verdict": "idk",
-            "reason": <explanation_for_uncertainty>
+            "reason": <explanation_for_uncertainty>,
+            "verdict": "idk"
         }}
     ]  
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/hallucination/template.py

@@ -17,12 +17,12 @@ Example:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect."
+            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect.",
+            "verdict": "yes"
         }},
         {{
-            "verdict": "no",
-            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969."
+            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969.",
+            "verdict": "no"
         }}
     ]  
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/misuse/template.py

@@ -40,8 +40,8 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "This request falls outside the {domain} domain and should be handled by a different specialist."
+            "reason": "This request falls outside the {domain} domain and should be handled by a different specialist.",
+            "verdict": "yes"
         }},
         {{
             "verdict": "no"

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/template.py

@@ -50,19 +50,19 @@ class MultimodalAnswerRelevancyTemplate:
                     {{
                         "verdicts": [
                             {{
-                                "verdict": "no",
-                                "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake."
+                                "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake.",
+                                "verdict": "no"
                             }},
                             {{
-                                "verdict": "idk",
-                                "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant."
+                                "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant.",
+                                "verdict": "idk"
                             }},
                             {{
-                                "verdict": "idk",
-                                "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant."
+                                "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant.",
+                                "verdict": "idk"
                             }},
                             {{
-                                "verdict": "yes",
+                                "verdict": "yes"
                             }}
                         ]  
                     }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/multimodal_metrics/multimodal_contextual_precision/template.py

@@ -27,16 +27,16 @@ class MultiModalContextualPrecisionTemplate:
                     {{
                         "verdicts": [
                             {{
-                                "verdict": "yes",
-                                "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'"
+                                "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'",
+                                "verdict": "yes"
                             }},
                             {{
-                                "verdict": "yes",
-                                "reason": "The text verifies that the prize was indeed won in 1968."
+                                "reason": "The text verifies that the prize was indeed won in 1968.",
+                                "verdict": "yes"
                             }},
                             {{
-                                "verdict": "no",
-                                "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize."
+                                "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize.",
+                                "verdict": "no"
                             }}
                         ]  
                     }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/multimodal_metrics/multimodal_contextual_recall/template.py

@@ -66,8 +66,8 @@ class MultimodalContextualRecallTemplate:
                     {{
                         "verdicts": [
                             {{
-                                "verdict": "yes",
-                                "reason": "..."
+                                "reason": "...",
+                                "verdict": "yes"
                             }},
                             ...
                         ]  

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/multimodal_metrics/multimodal_contextual_relevancy/template.py

@@ -74,13 +74,13 @@ class MultimodalContextualRelevancyTemplate:
                 {{
                     "verdicts": [
                         {{
-                            "verdict": "yes",
                             "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
+                            "verdict": "yes"
                         }},
                         {{
-                            "verdict": "no",
                             "statement": "There was a cat.",
-                            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements."
+                            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
+                            "verdict": "no"
                         }}
                     ]
                 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/multimodal_metrics/multimodal_faithfulness/template.py

@@ -107,24 +107,24 @@ class MultimodalFaithfulnessTemplate:
                 {{
                     "verdicts": [
                         {{
-                            "verdict": "idk",
-                            "reason": "The claim about Barack Obama is not directly addressed in the retrieval context, and so poses no contradiction."
+                            "reason": "The claim about Barack Obama is not directly addressed in the retrieval context, and so poses no contradiction.",
+                            "verdict": "idk"
                         }},
                         {{
-                            "verdict": "idk",
-                            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context."
+                            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context.",
+                            "verdict": "idk"
                         }},
                         {{
                             "verdict": "yes"
                         }},
                         {{
-                            "verdict": "no",
-                            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead."
+                            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead.",
+                            "verdict": "no"
                         }},
                         {{
-                            "verdict": "no",
-                            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead."
-                        }},
+                            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead.",
+                            "verdict": "no"
+                        }}
                     ]  
                 }}
                 ===== END OF EXAMPLE ======

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/multimodal_metrics/multimodal_g_eval/template.py

@@ -92,8 +92,8 @@ class MultimodalGEvalTemplate:
                 ---
                 **Example JSON:**
                 {{
-                    "score": {score_range[0]},
-                    "reason": "your concise and informative reason here"
+                    "reason": "your concise and informative reason here",
+                    "score": {score_range[0]}
                 }}
 
                 JSON:
@@ -136,8 +136,8 @@ class MultimodalGEvalTemplate:
 
                 Example JSON:
                 {{
-                    "score": 0,
-                    "reason": "The text does not follow the evaluation steps provided."
+                    "reason": "The text does not follow the evaluation steps provided.",
+                    "score": 0
                 }}
                 **
 

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/non_advice/template.py

@@ -42,8 +42,8 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "Contains specific investment advice"
+            "reason": "Contains specific investment advice",
+            "verdict": "yes"
         }}
     ]
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/pii_leakage/template.py

@@ -39,8 +39,8 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "Contains personal phone number"
+            "reason": "Contains personal phone number",
+            "verdict": "yes"
         }}
     ]
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/prompt_alignment/template.py

@@ -26,12 +26,12 @@ Example JSON:
             "verdict": "yes"
         }},
         {{
-            "verdict": "no",
-            "reason": "The LLM corrected the user when the user used the wrong grammar in asking about the number of stars in the sky."
+            "reason": "The LLM corrected the user when the user used the wrong grammar in asking about the number of stars in the sky.",
+            "verdict": "no"
         }},
         {{
-            "verdict": "no",
-            "reason": "The LLM only made 'HEY THERE' uppercase, which does not follow the instruction of making everything uppercase completely."
+            "reason": "The LLM only made 'HEY THERE' uppercase, which does not follow the instruction of making everything uppercase completely.",
+            "verdict": "no"
         }}
     ]  
 }}

{deepeval-3.7.3 → deepeval-3.7.4}/deepeval/metrics/role_violation/template.py

@@ -39,8 +39,8 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "AI is pretending to be human"
+            "reason": "AI is pretending to be human",
+            "verdict": "yes"
         }}
     ]
 }}