deepeval 3.4.8__py3-none-any.whl → 3.4.9__py3-none-any.whl

deepeval/config/settings_manager.py

@@ -0,0 +1,133 @@
+"""
+Applies CLI driven updates to the live Settings and optionally persists them to a
+dotenv file. Also syncs os.environ, handles unsets, and warns on unknown fields.
+Primary entrypoint: update_settings_and_persist.
+"""
+
+import logging
+import os
+
+from difflib import get_close_matches
+from pathlib import Path
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
+from enum import Enum
+
+from pydantic import SecretStr
+from deepeval.config.settings import get_settings, _SAVE_RE
+from deepeval.cli.dotenv_handler import DotenvHandler
+from deepeval.utils import bool_to_env_str
+
+logger = logging.getLogger(__name__)
+StrOrEnum = Union[str, Enum]
+
+
+def _env_key(k: StrOrEnum) -> str:
+    return k.value if isinstance(k, Enum) else str(k)
+
+
+def _normalize_for_env(val: Any) -> Optional[str]:
+    """Convert typed value to string for dotenv + os.environ; None -> unset."""
+    if val is None:
+        return None
+    if isinstance(val, SecretStr):
+        return val.get_secret_value()
+    if isinstance(val, bool):
+        return bool_to_env_str(val)
+    return str(val)
+
+
+def _resolve_save_path(save_opt: Optional[str]) -> Tuple[bool, Optional[Path]]:
+    """
+    Returns (ok, path).
+      - ok=False -> invalid save option format
+      - ok=True, path=None -> no persistence requested
+      - ok=True, path=Path -> persist to that file
+    """
+    raw = (
+        save_opt if save_opt is not None else os.getenv("DEEPEVAL_DEFAULT_SAVE")
+    )
+    if not raw:
+        return True, None
+    m = _SAVE_RE.match(raw.strip())
+    if not m:
+        return False, None
+    path = m.group("path") or ".env.local"
+    path = Path(os.path.expanduser(os.path.expandvars(path)))
+    return True, path
+
+
+def update_settings_and_persist(
+    updates: Mapping[StrOrEnum, Any],
+    *,
+    save: Optional[str] = None,
+    unset: Iterable[StrOrEnum] = (),
+    persist_dotenv: bool = True,
+) -> Tuple[bool, Optional[Path]]:
+    """
+    Write and update:
+      - validate + assign into live Settings()
+      - update os.environ
+      - persist to dotenv, if `save` or DEEPEVAL_DEFAULT_SAVE provided
+      - unset keys where value is None or explicitly in `unset`
+    Returns (handled, path_to_dotenv_if_any).
+    """
+    settings = get_settings()
+
+    # validate + assign into settings.
+    # validation is handled in Settings as long as validate_assignment=True
+    typed: Dict[str, Any] = {}
+    for key, value in updates.items():
+        k = _env_key(key)
+        if k not in type(settings).model_fields:
+            suggestion = get_close_matches(
+                k, type(settings).model_fields.keys(), n=1
+            )
+            if suggestion:
+                logger.warning(
+                    "Unknown settings field '%s'; did you mean '%s'? Ignoring.",
+                    k,
+                    suggestion[0],
+                    stacklevel=2,
+                )
+            else:
+                logger.warning(
+                    "Unknown settings field '%s'; ignoring.", k, stacklevel=2
+                )
+            continue
+
+        setattr(settings, k, value)
+        # coercion is handled in Settings
+        typed[k] = getattr(settings, k)
+
+    # build env maps
+    to_write: Dict[str, str] = {}
+    to_unset: set[str] = set(_env_key(k) for k in unset)
+
+    for k, v in typed.items():
+        env_val = _normalize_for_env(v)
+        if env_val is None:
+            to_unset.add(k)
+        else:
+            to_write[k] = env_val
+
+    # update process env so that it is effective immediately
+    for k, v in to_write.items():
+        os.environ[k] = v
+    for k in to_unset:
+        os.environ.pop(k, None)
+
+    if not persist_dotenv:
+        return True, None
+
+    # persist to dotenv if save is ok
+    ok, path = _resolve_save_path(save)
+    if not ok:
+        return False, None  # unsupported --save
+    if path:
+        h = DotenvHandler(path)
+        if to_write:
+            h.upsert(to_write)
+        if to_unset:
+            h.unset(to_unset)
+        return True, path
+    return True, None

deepeval/config/utils.py

@@ -0,0 +1,86 @@
+import os
+from typing import Any, Optional
+
+_TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on", "enable", "enabled"})
+_FALSY = frozenset({"0", "false", "f", "no", "n", "off", "disable", "disabled"})
+
+
+def parse_bool(value: Any, default: bool = False) -> bool:
+    """
+    Parse an arbitrary value into a boolean using env style semantics.
+
+    Truthy tokens (case-insensitive, quotes/whitespace ignored):
+      1, true, t, yes, y, on, enable, enabled
+    Falsy tokens:
+      0, false, f, no, n, off, disable, disabled
+
+    - bool -> returned as is
+    - None -> returns `default`
+    - int/float -> False if == 0, else True
+    - str/other -> matched against tokens above; non-matching -> `default`
+
+    Args:
+        value: Value to interpret.
+        default: Value to return if `value` is None or doesn’t match any token.
+
+    Returns:
+        The interpreted boolean.
+    """
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return default
+    if isinstance(value, (int, float)):
+        return value != 0
+
+    s = str(value).strip().strip('"').strip("'").lower()
+    if not s:
+        return default
+    if s in _TRUTHY:
+        return True
+    if s in _FALSY:
+        return False
+    return default
+
+
+def get_env_bool(key: str, default: bool = False) -> bool:
+    """
+    Read an environment variable and parse it as a boolean using `parse_bool`.
+
+    Args:
+        key: Environment variable name.
+        default: Returned when the variable is unset or does not match any token.
+
+    Returns:
+        Parsed boolean value.
+    """
+    return parse_bool(os.getenv(key), default)
+
+
+def bool_to_env_str(value: bool) -> str:
+    """
+    Canonicalize a boolean to the env/dotenv string form: "1" or "0".
+
+    Args:
+        value: Boolean to serialize.
+
+    Returns:
+        "1" if True, "0" if False.
+    """
+    return "1" if bool(value) else "0"
+
+
+def set_env_bool(key: str, value: Optional[bool] = False) -> None:
+    """
+    Set an environment variable to a canonical boolean string ("1" or "0").
+
+    Args:
+        key: The environment variable name to set.
+        value: The boolean value to store. If None, it is treated as False.
+               True -> "1", False/None -> "0".
+
+    Notes:
+        - This function always overwrites the variable in `os.environ`.
+        - Use `get_env_bool` to read back and parse the value safely.
+    """
+    os.environ[key] = bool_to_env_str(bool(value))

deepeval/dataset/__init__.py

@@ -1,4 +1,5 @@
 from .dataset import EvaluationDataset
 from .golden import Golden, ConversationalGolden
+from .test_run_tracer import init_global_test_run_tracer
 
 __all__ = ["EvaluationDataset", "Golden", "ConversationalGolden"]

deepeval/dataset/dataset.py

@@ -1,6 +1,8 @@
 from asyncio import Task
 from typing import Iterator, List, Optional, Union, Literal
 from dataclasses import dataclass, field
+from opentelemetry.trace import Tracer
+from opentelemetry.context import Context, attach, detach
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 import json
@@ -10,6 +12,8 @@ import os
 import datetime
 import time
 import ast
+import uuid
+from opentelemetry import baggage
 
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.dataset.utils import (
@@ -18,6 +22,7 @@ from deepeval.dataset.utils import (
     convert_convo_goldens_to_convo_test_cases,
     convert_convo_test_cases_to_convo_goldens,
     format_turns,
+    check_tracer,
     parse_turns,
     trimAndLoadJson,
 )
@@ -47,6 +52,7 @@ from deepeval.test_run import (
 from deepeval.dataset.types import global_evaluation_tasks
 from deepeval.openai.utils import openai_test_case_pairs
 from deepeval.tracing import trace_manager
+from deepeval.tracing.tracing import EVAL_DUMMY_SPAN_NAME
 
 
 valid_file_types = ["csv", "json", "jsonl"]
@@ -1097,6 +1103,7 @@ class EvaluationDataset:
         cache_config: Optional["CacheConfig"] = None,
         error_config: Optional["ErrorConfig"] = None,
         async_config: Optional["AsyncConfig"] = None,
+        run_otel: Optional[bool] = False,
     ) -> Iterator[Golden]:
         from deepeval.evaluate.utils import (
             aggregate_metric_pass_rates,
@@ -1133,9 +1140,14 @@ class EvaluationDataset:
             start_time = time.perf_counter()
             test_results: List[TestResult] = []
 
+            # sandwich start trace for OTEL
+            if run_otel:
+                ctx = self._start_otel_test_run()  # ignored span
+                ctx_token = attach(ctx)
+
             if async_config.run_async:
                 loop = get_or_create_event_loop()
-                yield from a_execute_agentic_test_cases_from_loop(
+                for golden in a_execute_agentic_test_cases_from_loop(
                     goldens=goldens,
                     identifier=identifier,
                     loop=loop,
@@ -1145,9 +1157,19 @@ class EvaluationDataset:
                     cache_config=cache_config,
                     error_config=error_config,
                     async_config=async_config,
-                )
+                ):
+                    if run_otel:
+                        _tracer = check_tracer()
+                        with _tracer.start_as_current_span(
+                            name=EVAL_DUMMY_SPAN_NAME,
+                            context=ctx,
+                        ):
+                            yield golden
+                    else:
+                        yield golden
+
             else:
-                yield from execute_agentic_test_cases_from_loop(
+                for golden in execute_agentic_test_cases_from_loop(
                     goldens=goldens,
                     trace_metrics=metrics,
                     display_config=display_config,
@@ -1155,7 +1177,16 @@ class EvaluationDataset:
                     error_config=error_config,
                     test_results=test_results,
                     identifier=identifier,
-                )
+                ):
+                    if run_otel:
+                        _tracer = check_tracer()
+                        with _tracer.start_as_current_span(
+                            name=EVAL_DUMMY_SPAN_NAME,
+                            context=ctx,
+                        ):
+                            yield golden
+                    else:
+                        yield golden
 
             end_time = time.perf_counter()
             run_duration = end_time - start_time
@@ -1184,12 +1215,41 @@ class EvaluationDataset:
             # clean up
             openai_test_case_pairs.clear()
             global_test_run_manager.save_test_run(TEMP_FILE_PATH)
-            confident_link = global_test_run_manager.wrap_up_test_run(
-                run_duration, display_table=False
-            )
-            return EvaluationResult(
-                test_results=test_results, confident_link=confident_link
-            )
+
+            # sandwich end trace for OTEL
+            if run_otel:
+                self._end_otel_test_run(ctx)
+                detach(ctx_token)
+
+            else:
+                confident_link = global_test_run_manager.wrap_up_test_run(
+                    run_duration, display_table=False
+                )
+                return EvaluationResult(
+                    test_results=test_results, confident_link=confident_link
+                )
 
     def evaluate(self, task: Task):
         global_evaluation_tasks.append(task)
+
+    def _start_otel_test_run(self, tracer: Optional[Tracer] = None) -> Context:
+        _tracer = check_tracer(tracer)
+        run_id = str(uuid.uuid4())
+        print("Starting OTLP test run with run_id: ", run_id)
+        ctx = baggage.set_baggage(
+            "confident.test_run.id", run_id, context=Context()
+        )
+        with _tracer.start_as_current_span(
+            "start_otel_test_run", context=ctx
+        ) as span:
+            span.set_attribute("confident.test_run.id", run_id)
+        return ctx
+
+    def _end_otel_test_run(self, ctx: Context, tracer: Optional[Tracer] = None):
+        run_id = baggage.get_baggage("confident.test_run.id", context=ctx)
+        print("Ending OTLP test run with run_id: ", run_id)
+        _tracer = check_tracer(tracer)
+        with _tracer.start_as_current_span(
+            "stop_otel_test_run", context=ctx
+        ) as span:
+            span.set_attribute("confident.test_run.id", run_id)

deepeval/dataset/test_run_tracer.py

@@ -0,0 +1,82 @@
+import os
+from typing import Optional
+from opentelemetry import baggage
+from opentelemetry.trace import Tracer as OTelTracer
+from opentelemetry.sdk.trace import SpanProcessor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+try:
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+
+    is_opentelemetry_installed = True
+except Exception:
+    is_opentelemetry_installed = False
+
+
+def is_opentelemetry_available():
+    if not is_opentelemetry_installed:
+        raise ImportError(
+            "OpenTelemetry SDK is not available. Please install it with `pip install opentelemetry-exporter-otlp-proto-http`."
+        )
+    return True
+
+
+from deepeval.confident.api import get_confident_api_key
+
+OTLP_ENDPOINT = (
+    os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+    else "https://otel.confident-ai.com"
+)
+# OTLP_ENDPOINT = "http://127.0.0.1:4318"
+
+# Module-level globals to be imported and used by other code
+GLOBAL_TEST_RUN_TRACER_PROVIDER: Optional[TracerProvider] = None
+GLOBAL_TEST_RUN_TRACER: Optional[OTelTracer] = None
+
+
+class RunIdSpanProcessor(SpanProcessor):
+    def on_start(self, span, parent_context):
+        run_id = baggage.get_baggage(
+            "confident.test_run.id", context=parent_context
+        )
+        if run_id:
+            span.set_attribute("confident.test_run.id", run_id)
+
+    def on_end(self, span) -> None:  # type: ignore[override]
+        # No-op
+        return None
+
+    def shutdown(self) -> None:  # type: ignore[override]
+        # No-op
+        return None
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:  # type: ignore[override]
+        # No-op
+        return True
+
+
+def init_global_test_run_tracer(api_key: Optional[str] = None):
+    is_opentelemetry_available()
+    api_key = get_confident_api_key()
+    if api_key is None:
+        raise ValueError("CONFIDENT_API_KEY is not set")
+
+    provider = TracerProvider()
+    exporter = OTLPSpanExporter(
+        endpoint=f"{OTLP_ENDPOINT}/v1/traces",
+        headers={"x-confident-api-key": api_key},
+    )
+    provider.add_span_processor(RunIdSpanProcessor())
+    provider.add_span_processor(BatchSpanProcessor(span_exporter=exporter))
+    tracer = provider.get_tracer("deepeval_tracer")
+
+    global GLOBAL_TEST_RUN_TRACER_PROVIDER
+    global GLOBAL_TEST_RUN_TRACER
+    GLOBAL_TEST_RUN_TRACER_PROVIDER = provider
+    GLOBAL_TEST_RUN_TRACER = tracer
+
+    return provider, tracer

deepeval/dataset/utils.py

@@ -2,6 +2,10 @@ from typing import List, Optional, Any
 import json
 import re
 
+from opentelemetry.trace import Tracer
+from opentelemetry import trace
+from opentelemetry.trace import NoOpTracerProvider
+
 from deepeval.dataset.api import Golden
 from deepeval.dataset.golden import ConversationalGolden
 from deepeval.test_case import LLMTestCase, ConversationalTestCase, Turn
@@ -151,3 +155,22 @@ def parse_turns(turns_str: str) -> List[Turn]:
             )
         )
     return res
+
+
+def check_tracer(tracer: Optional[Tracer] = None) -> Tracer:
+    if tracer:
+        return tracer
+    # Prefer module-level test-run tracer if available
+    try:
+        from deepeval.dataset.test_run_tracer import (
+            GLOBAL_TEST_RUN_TRACER,
+        )
+
+        if GLOBAL_TEST_RUN_TRACER is not None:
+            return GLOBAL_TEST_RUN_TRACER
+    except Exception:
+        raise RuntimeError(
+            "No global OpenTelemetry tracer provider is configured."  # TODO: link to docs
+        )
+
+    return GLOBAL_TEST_RUN_TRACER

deepeval/key_handler.py

@@ -80,6 +80,7 @@ class ModelKeyValues(Enum):
     OPENAI_MODEL_NAME = "OPENAI_MODEL_NAME"
     OPENAI_COST_PER_INPUT_TOKEN = "OPENAI_COST_PER_INPUT_TOKEN"
     OPENAI_COST_PER_OUTPUT_TOKEN = "OPENAI_COST_PER_OUTPUT_TOKEN"
+    OPENAI_API_KEY = "OPENAI_API_KEY"
     # Moonshot
     USE_MOONSHOT_MODEL = "USE_MOONSHOT_MODEL"
     MOONSHOT_MODEL_NAME = "MOONSHOT_MODEL_NAME"

deepeval/metrics/answer_relevancy/template.py

@@ -37,7 +37,7 @@ JSON:
 Please generate a list of JSON with two keys: `verdict` and `reason`.
 The 'verdict' key should STRICTLY be either a 'yes', 'idk' or 'no'. Answer 'yes' if the statement is relevant to addressing the original input, 'no' if the statement is irrelevant, and 'idk' if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).
 The 'reason' is the reason for the verdict.
-Provide a 'reason' ONLY if the answer is 'no'. 
+Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
 The provided statements are statements made in the actual output.
 
 **
@@ -53,7 +53,8 @@ Example statements:
     "Security features include fingerprint authentication and an encrypted SSD.",
     "Every purchase comes with a one-year warranty.",
     "24/7 customer support is included.",
-    "Pineapples taste great on pizza."
+    "Pineapples taste great on pizza.",
+    "The laptop is a Dell XPS 13."
 ]
 
 Example JSON:
@@ -79,6 +80,10 @@ Example JSON:
         {{
             "verdict": "no",
             "reason": "The statement about pineapples on pizza is completely irrelevant to the input, which asks about laptop features."
+        }},
+        {{
+            "verdict": "idk",
+            "reason": "The statement about the laptop being a Dell XPS 13 is not directly relevant to the input, but could be used as a supporting point to address the input."
         }}
     ]  
 }}

deepeval/metrics/faithfulness/template.py

@@ -4,7 +4,7 @@ from typing import Optional, List
 class FaithfulnessTemplate:
     @staticmethod
     def generate_claims(actual_output: str):
-        return f"""Based on the given text, please extract a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided text.
+        return f"""Based on the given text, please extract a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided actual AI output.
 These truths, MUST BE COHERENT, and CANNOT be taken out of context.
     
 Example:
@@ -24,9 +24,10 @@ Example JSON:
 IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
 Only include claims that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT. The claims you extract should include the full context it was presented in, NOT cherry picked facts.
 You should NOT include any prior knowledge, and take the text at face value when extracting claims.
+You should be aware that it is an AI that is outputting these claims.
 **
 
-Text:
+AI Output:
 {actual_output}
 
 JSON:
@@ -72,7 +73,7 @@ JSON:
     def generate_verdicts(claims: List[str], retrieval_context: str):
         return f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
 The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
-Provide a 'reason' ONLY if the answer is 'no'. 
+Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
 The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
 
 **
@@ -84,28 +85,30 @@ Example:
 {{
     "verdicts": [
         {{
-            "verdict": "idk"
+            "verdict": "idk",
+            "reason": "The claim about Barack Obama is although incorrect, it is not directly addressed in the retrieval context, and so poses no contradiction."
         }},
         {{
-            "verdict": "idk"
+            "verdict": "idk",
+            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context."
         }},
         {{
             "verdict": "yes"
         }},
         {{
             "verdict": "no",
-            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead."
+            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead. This contradicts the retrieval context."
         }},
         {{
             "verdict": "no",
-            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead."
+            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead. This contradicts the retrieval context."
         }},
     ]  
 }}
 ===== END OF EXAMPLE ======
 
 The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
-You DON'T have to provide a reason if the answer is 'yes' or 'idk'.
+You DON'T have to provide a reason if the answer is 'yes'.
 ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
 Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
 Claims that are not backed up by the retrieval context or are not mentioned in it MUST be answered 'idk'.

deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/template.py

@@ -39,7 +39,7 @@ class MultimodalAnswerRelevancyTemplate:
                     Please generate a list of JSON with two keys: `verdict` and `reason`.
                     The 'verdict' key should STRICTLY be either a 'yes', 'idk' or 'no'. Answer 'yes' if the statement or image is relevant to addressing the original input, 'no' if the statement or image is irrelevant, and 'idk' if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).
                     The 'reason' is the reason for the verdict.
-                    Provide a 'reason' ONLY if the answer is 'no'. 
+                    Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
                     The provided statements are statements and images generated in the actual output.
 
                     **
@@ -54,13 +54,15 @@ class MultimodalAnswerRelevancyTemplate:
                                 "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake."
                             }},
                             {{
-                                "verdict": "idk"
+                                "verdict": "idk",
+                                "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant."
                             }},
                             {{
-                                "verdict": "idk"
+                                "verdict": "idk",
+                                "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant."
                             }},
                             {{
-                                "verdict": "yes"
+                                "verdict": "yes",
                             }}
                         ]  
                     }}

deepeval/metrics/multimodal_metrics/multimodal_faithfulness/template.py

@@ -95,7 +95,7 @@ class MultimodalFaithfulnessTemplate:
         return textwrap.dedent(
             f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
                 The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
-                Provide a 'reason' ONLY if the answer is 'no'. 
+                Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
                 The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
 
                 **
@@ -107,10 +107,12 @@ class MultimodalFaithfulnessTemplate:
                 {{
                     "verdicts": [
                         {{
-                            "verdict": "idk"
+                            "verdict": "idk",
+                            "reason": "The claim about Barack Obama is not directly addressed in the retrieval context, and so poses no contradiction."
                         }},
                         {{
-                            "verdict": "idk"
+                            "verdict": "idk",
+                            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context."
                         }},
                         {{
                             "verdict": "yes"
@@ -128,7 +130,7 @@ class MultimodalFaithfulnessTemplate:
                 ===== END OF EXAMPLE ======
 
                 The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
-                You DON'T have to provide a reason if the answer is 'yes' or 'idk'.
+                You DON'T have to provide a reason if the answer is 'yes'.
                 ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
                 Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
                 Claims that is not backed up due to a lack of information/is not mentioned in the retrieval contexts MUST be answered 'idk', otherwise I WILL DIE.

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -223,9 +223,13 @@ class ToolCorrectnessMetric(BaseMetric):
                 total_score += best_score
                 matched_called_tools.add(best_called_tool)
         return (
-            total_score / len(self.expected_tools)
-            if self.expected_tools
-            else 0.0
+            1.0
+            if not self.expected_tools and not self.tools_called
+            else (
+                0.0
+                if not self.expected_tools
+                else total_score / len(self.expected_tools)
+            )
         )
 
     # Consider ordering score