deepeval 4.2.2__tar.gz → 4.2.4__tar.gz

{deepeval-4.2.2 → deepeval-4.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepeval
-Version: 4.2.2
+Version: 4.2.4
 Summary: The LLM Evaluation Framework
 License: Apache-2.0
 License-File: LICENSE.md

deepeval-4.2.4/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "4.2.4"

deepeval-4.2.4/deepeval/classifiers/__init__.py

@@ -0,0 +1,36 @@
+from .base_classifier import BaseClassifier, Label
+from .classifier import Classifier, ClassifierTemplate
+
+from .refusal import RefusalClassifier
+from .escalation import EscalationClassifier
+from .scope_adherence import ScopeAdherenceClassifier
+from .clarification import ClarificationClassifier
+from .abstention import AbstentionClassifier
+from .prompt_injection import PromptInjectionClassifier
+from .data_leakage import DataLeakageClassifier
+from .forbidden_commitments import ForbiddenCommitmentsClassifier
+from .tone_adherence import ToneAdherenceClassifier
+from .required_disclosure import RequiredDisclosureClassifier
+from .response_language import ResponseLanguageClassifier
+from .instruction_completeness import InstructionCompletenessClassifier
+from .resolution import ResolutionClassifier
+
+__all__ = [
+    "Classifier",
+    "Label",
+    "BaseClassifier",
+    "ClassifierTemplate",
+    "RefusalClassifier",
+    "EscalationClassifier",
+    "ScopeAdherenceClassifier",
+    "ClarificationClassifier",
+    "AbstentionClassifier",
+    "PromptInjectionClassifier",
+    "DataLeakageClassifier",
+    "ForbiddenCommitmentsClassifier",
+    "ToneAdherenceClassifier",
+    "RequiredDisclosureClassifier",
+    "ResponseLanguageClassifier",
+    "InstructionCompletenessClassifier",
+    "ResolutionClassifier",
+]

deepeval-4.2.4/deepeval/classifiers/abstention.py

@@ -0,0 +1,42 @@
+from typing import List, Optional, Type, Union
+
+from deepeval.classifiers.base_classifier import Label
+from deepeval.classifiers.classifier import Classifier, ClassifierTemplate
+from deepeval.models import DeepEvalBaseLLM
+
+DEFAULT_LABELS: List[Label] = [
+    Label(
+        "abstained",
+        "The provided context does not contain the answer and the response says so instead of answering.",
+    ),
+    Label(
+        "answered",
+        "The provided context contains the answer and the response gives it.",
+    ),
+    Label(
+        "fabricated",
+        "The provided context does not contain the answer but the response gives one anyway.",
+    ),
+]
+
+
+class AbstentionClassifier(Classifier):
+    """Labels whether the assistant abstained, answered from context, or fabricated when context lacked the answer."""
+
+    def __init__(
+        self,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        allow_none: bool = False,
+        async_mode: bool = True,
+        classification_template: Type[ClassifierTemplate] = ClassifierTemplate,
+    ):
+        super().__init__(
+            name="abstention",
+            labels=list(DEFAULT_LABELS),
+            model=model,
+            include_reason=include_reason,
+            allow_none=allow_none,
+            async_mode=async_mode,
+            classification_template=classification_template,
+        )

deepeval-4.2.4/deepeval/classifiers/base_classifier.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, List, Optional, Union
+
+from pydantic import BaseModel
+
+from deepeval.metrics.base_metric import PromptMixin
+from deepeval.test_case import ConversationalTestCase, LLMTestCase
+
+if TYPE_CHECKING:
+    from deepeval.models import DeepEvalBaseLLM
+    from deepeval.models.base_model import DeepEvalBaseSystemOneModel
+
+
+# Sentinel label the judge returns when none of the declared labels fit.
+NONE_LABEL = "NONE"
+
+
+class Label(BaseModel):
+    """A label a classifier can assign. The description is the boundary for
+    the label, the same way G-Eval criteria are the boundary for a score."""
+
+    name: str
+    description: Optional[str] = None
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        **kwargs,
+    ):
+        if name is not None:
+            kwargs["name"] = name
+        if description is not None:
+            kwargs["description"] = description
+        super().__init__(**kwargs)
+
+    # `print(classifier.labels)` should read as the list of label names a
+    # test case can put in `expected_labels`, not as pydantic reprs, and
+    # `"refused" in classifier.labels` should work.
+    def __repr__(self) -> str:
+        return repr(self.name)
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Label):
+            return (
+                self.name == other.name
+                and self.description == other.description
+            )
+        if isinstance(other, str):
+            return self.name == other
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash((self.name, self.description))
+
+
+class BaseClassifier(PromptMixin):
+    """Classifier counterpart of ``BaseMetric``.
+
+    A classifier assigns exactly one label from a closed set to a test case.
+    It only produces a pass/fail verdict when the test case declares an
+    expected label for it (``test_case.expected_labels[classifier.name]``);
+    otherwise ``success`` is ``None``, mirroring a metric with no threshold.
+    """
+
+    _template_feature = "classifiers"
+    _template_attr = "classification_template"
+
+    name: str
+    labels: List[Label]
+    label: Optional[str] = None
+    reason: Optional[str] = None
+    success: Optional[bool] = None
+    evaluation_model: Optional[str] = None
+    # Kept only so shared helpers written for metrics (e.g. the progress
+    # indicator description) can read it. Classifiers have no strict mode.
+    strict_mode: bool = False
+    async_mode: bool = True
+    include_reason: bool = True
+    # When True the judge may decline to classify, surfaced as `label is None`
+    # with no error. When False every response must receive a label.
+    allow_none: bool = False
+    error: Optional[str] = None
+    evaluation_cost: Optional[float] = None
+    input_tokens: Optional[int] = None
+    output_tokens: Optional[int] = None
+    skipped = False
+    model: Optional[DeepEvalBaseLLM] = None
+    using_native_model: Optional[bool] = None
+    # Experimental (DEEPEVAL_MODE=experimental); see EXPERIMENTAL.md.
+    system_one_model: Optional[DeepEvalBaseSystemOneModel] = None
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        from deepeval.tracing.internal import observe_methods
+
+        observe_methods(cls)
+
+    @abstractmethod
+    def classify(
+        self,
+        test_case: Union[LLMTestCase, ConversationalTestCase],
+        *args,
+        **kwargs,
+    ) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_classify(
+        self,
+        test_case: Union[LLMTestCase, ConversationalTestCase],
+        *args,
+        **kwargs,
+    ) -> str:
+        raise NotImplementedError(
+            f"Async execution for {self.__class__.__name__} not supported yet. Please set 'async_mode' to 'False'."
+        )
+
+    def is_successful(
+        self, expected_label: Optional[str] = None
+    ) -> Optional[bool]:
+        if expected_label is None:
+            self.success = None
+        elif self.error is not None:
+            self.success = False
+        else:
+            self.success = self.label == expected_label
+        return self.success
+
+    def copy(self) -> "BaseClassifier":
+        """Return a fresh instance with the same configuration.
+
+        Mirrors ``copy_metrics``: the executor evaluates test cases
+        concurrently and each needs its own instance so per-case state
+        (``label``, ``reason``, ``error``, cost) never bleeds across cases.
+        Only the concrete class's own ``__init__`` parameters are replayed,
+        so built-in classifiers with fixed labels reconstruct cleanly.
+        """
+        import inspect
+
+        params = inspect.signature(type(self).__init__).parameters
+        state = vars(self)
+        kwargs = {
+            name: state[name]
+            for name, param in params.items()
+            if name != "self"
+            and param.kind
+            in (
+                inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                inspect.Parameter.KEYWORD_ONLY,
+            )
+            and name in state
+        }
+        return type(self)(**kwargs)
+
+    @property
+    def __name__(self):
+        return self.name
+
+    def _accrue_cost(self, cost: Optional[float]) -> None:
+        effective = getattr(cost, "value", cost)
+        if self.evaluation_cost is not None and effective is not None:
+            self.evaluation_cost += effective
+        else:
+            self.evaluation_cost = None
+
+    def _accrue_tokens(
+        self,
+        input_tokens: Optional[int],
+        output_tokens: Optional[int],
+    ) -> None:
+        if input_tokens is not None:
+            self.input_tokens = (self.input_tokens or 0) + input_tokens
+        if output_tokens is not None:
+            self.output_tokens = (self.output_tokens or 0) + output_tokens

deepeval-4.2.4/deepeval/classifiers/clarification.py

@@ -0,0 +1,42 @@
+from typing import List, Optional, Type, Union
+
+from deepeval.classifiers.base_classifier import Label
+from deepeval.classifiers.classifier import Classifier, ClassifierTemplate
+from deepeval.models import DeepEvalBaseLLM
+
+DEFAULT_LABELS: List[Label] = [
+    Label(
+        "asked_clarification",
+        "The input is ambiguous or underspecified and the response asks a clarifying question before proceeding.",
+    ),
+    Label(
+        "answered_directly",
+        "The input is clear enough and the response answers or acts on it without needing clarification.",
+    ),
+    Label(
+        "guessed",
+        "The input is ambiguous or underspecified but the response proceeds on an assumption instead of asking.",
+    ),
+]
+
+
+class ClarificationClassifier(Classifier):
+    """Labels whether the assistant asked for clarification, answered directly, or guessed on ambiguous input."""
+
+    def __init__(
+        self,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        allow_none: bool = False,
+        async_mode: bool = True,
+        classification_template: Type[ClassifierTemplate] = ClassifierTemplate,
+    ):
+        super().__init__(
+            name="clarification",
+            labels=list(DEFAULT_LABELS),
+            model=model,
+            include_reason=include_reason,
+            allow_none=allow_none,
+            async_mode=async_mode,
+            classification_template=classification_template,
+        )

deepeval-4.2.4/deepeval/classifiers/classifier/__init__.py

@@ -0,0 +1,3 @@
+from .classifier import Classifier, ClassifierTemplate
+
+__all__ = ["Classifier", "ClassifierTemplate"]

deepeval-4.2.4/deepeval/classifiers/classifier/classifier.py

@@ -0,0 +1,254 @@
+"""LLM-judged classifier: assigns exactly one label from a closed set."""
+
+import asyncio
+from typing import Dict, List, Optional, Sequence, Tuple, Type, Union
+
+from deepeval.classifiers.base_classifier import (
+    NONE_LABEL,
+    BaseClassifier,
+    Label,
+)
+from deepeval.classifiers.classifier.schema import (
+    ClassificationResult,
+    Reason,
+)
+from deepeval.classifiers.utils import (
+    SystemOneClassifySpec,
+    a_generate_classification,
+    construct_multi_turn_content,
+    construct_multi_turn_state,
+    construct_single_turn_content,
+    construct_single_turn_state,
+    construct_turns,
+    format_labels,
+    generate_classification,
+    normalize_labels,
+    resolve_label,
+)
+from deepeval.config.settings import get_settings
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    initialize_model,
+    initialize_system_one_model,
+)
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.templates import make_template_class
+from deepeval.test_case import ConversationalTestCase, LLMTestCase
+from deepeval.utils import get_or_create_event_loop
+
+ClassifierTemplate = make_template_class("Classifier", feature="classifiers")
+
+# Built-in classifiers (``RefusalClassifier`` ...) share ``Classifier``'s bundle
+# entry. Public templates reach it through ``classification_template``; the
+# ``_experimental_*`` ones are not exposed on the template class, so they name
+# the bundle key explicitly.
+_EXPERIMENTAL_TEMPLATE_CLASS = "Classifier"
+
+
+class Classifier(BaseClassifier):
+    def __init__(
+        self,
+        name: str,
+        labels: Sequence[Union[str, Label]],
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        allow_none: bool = False,
+        async_mode: bool = True,
+        classification_template: Type[ClassifierTemplate] = ClassifierTemplate,
+    ):
+        if not name or not name.strip():
+            raise ValueError("Classifier 'name' cannot be empty.")
+
+        self.name = name
+        self.labels = normalize_labels(labels)
+        self.model, self.using_native_model = initialize_model(model)
+        self.system_one_model = initialize_system_one_model()
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.allow_none = allow_none
+        self.async_mode = async_mode
+        self.classification_template = classification_template
+
+    def classify(
+        self,
+        test_case: Union[LLMTestCase, ConversationalTestCase],
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> str:
+        self._check_test_case(test_case)
+        self._reset_usage()
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                settings = get_settings()
+                loop.run_until_complete(
+                    asyncio.wait_for(
+                        self.a_classify(
+                            test_case,
+                            _show_indicator=False,
+                            _in_component=_in_component,
+                        ),
+                        timeout=(
+                            None
+                            if settings.DEEPEVAL_DISABLE_TIMEOUTS
+                            else settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+                        ),
+                    )
+                )
+            else:
+                prompt = self._build_prompt(test_case)
+                raw_label, reason = generate_classification(
+                    self,
+                    prompt,
+                    schema_cls=ClassificationResult,
+                    system_one=self._experimental_system_one_spec(test_case),
+                )
+                self._finalize(raw_label, reason)
+
+            return self.label
+
+    async def a_classify(
+        self,
+        test_case: Union[LLMTestCase, ConversationalTestCase],
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> str:
+        self._check_test_case(test_case)
+        self._reset_usage()
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            prompt = self._build_prompt(test_case)
+            raw_label, reason = await a_generate_classification(
+                self,
+                prompt,
+                schema_cls=ClassificationResult,
+                system_one=self._experimental_system_one_spec(test_case),
+            )
+            self._finalize(raw_label, reason)
+            return self.label
+
+    ###############################################
+    # Internals
+    ###############################################
+
+    def _check_test_case(
+        self, test_case: Union[LLMTestCase, ConversationalTestCase]
+    ) -> None:
+        if not isinstance(test_case, (LLMTestCase, ConversationalTestCase)):
+            raise TypeError(
+                f"{self.__class__.__name__} expects an LLMTestCase or ConversationalTestCase, got {type(test_case).__name__}."
+            )
+
+    def _reset_usage(self) -> None:
+        self.evaluation_cost = 0 if self.using_native_model else None
+        self.input_tokens = 0 if self.using_native_model else None
+        self.output_tokens = 0 if self.using_native_model else None
+        self.label = None
+        self.reason = None
+        self.error = None
+
+    def _build_prompt(
+        self, test_case: Union[LLMTestCase, ConversationalTestCase]
+    ) -> str:
+        labels = format_labels(self.labels)
+        if isinstance(test_case, ConversationalTestCase):
+            return self._get_prompt(
+                "classify_multi_turn",
+                labels=labels,
+                test_case_content=construct_multi_turn_content(test_case),
+                turns=construct_turns(test_case),
+                allow_none=self.allow_none,
+            )
+        return self._get_prompt(
+            "classify_single_turn",
+            labels=labels,
+            test_case_content=construct_single_turn_content(test_case),
+            allow_none=self.allow_none,
+        )
+
+    def _experimental_system_one_spec(
+        self, test_case: Union[LLMTestCase, ConversationalTestCase]
+    ) -> SystemOneClassifySpec:
+        """Experimental (DEEPEVAL_MODE=experimental); see EXPERIMENTAL.md.
+
+        The label is one Choice over the declared labels (plus ``NONE`` when
+        ``allow_none``); the LLM only writes the reason, if one is wanted.
+        """
+        multi_turn = isinstance(test_case, ConversationalTestCase)
+        state = (
+            construct_multi_turn_state(test_case)
+            if multi_turn
+            else construct_single_turn_state(test_case)
+        )
+        options: Dict[str, Optional[str]] = {
+            label.name: label.description for label in self.labels
+        }
+        if self.allow_none:
+            options[NONE_LABEL] = "None of the other labels apply."
+
+        reason_prompt = None
+        if self.include_reason:
+            labels = format_labels(self.labels)
+            test_case_content = (
+                construct_multi_turn_content(test_case)
+                if multi_turn
+                else construct_single_turn_content(test_case)
+            )
+            turns = construct_turns(test_case) if multi_turn else None
+
+            def reason_prompt(
+                label: str, probabilities: Dict[str, float], confidence: float
+            ) -> str:
+                return self._get_prompt(
+                    "_experimental_system_one_reason",
+                    template_class=_EXPERIMENTAL_TEMPLATE_CLASS,
+                    labels=labels,
+                    label=label,
+                    probabilities=probabilities,
+                    confidence=confidence,
+                    test_case_content=test_case_content,
+                    turns=turns,
+                    allow_none=self.allow_none,
+                )
+
+        return SystemOneClassifySpec(
+            instructions=self._get_prompt(
+                "_experimental_system_one_classify",
+                template_class=_EXPERIMENTAL_TEMPLATE_CLASS,
+                allow_none=self.allow_none,
+                multi_turn=multi_turn,
+            ),
+            options=options,
+            state=state,
+            reason_prompt=reason_prompt,
+            reason_schema_cls=Reason,
+        )
+
+    def _finalize(
+        self, raw_label: Optional[str], reason: Optional[str]
+    ) -> None:
+        label = resolve_label(raw_label, self.labels, self.allow_none)
+        if label is None:
+            allowed = ", ".join(str(label) for label in self.labels)
+            if self.allow_none:
+                allowed += ", or no classification"
+            self.error = (
+                f"Judge returned '{raw_label}', which is not one of the declared labels "
+                f"({allowed})."
+            )
+            self.label = None
+        elif label == NONE_LABEL:
+            # The judge declined to classify; that is a valid outcome when
+            # allow_none is on, surfaced to users as `label is None` with no error.
+            self.label = None
+        else:
+            self.label = label
+        self.reason = reason if self.include_reason else None

deepeval-4.2.4/deepeval/classifiers/classifier/schema.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+
+class ClassificationResult(BaseModel):
+    label: str
+    reason: str
+
+
+# Experimental (DEEPEVAL_MODE=experimental); see EXPERIMENTAL.md.
+class Reason(BaseModel):
+    reason: str

deepeval-4.2.4/deepeval/classifiers/classifier/templates/_experimental_system_one_classify.txt

@@ -0,0 +1 @@
+Which option best describes {% if multi_turn %}the conversation in `turns`{% else %}`test_case`{% endif %}? Each option is a label; where a description is given it is the authoritative boundary for that label.{% if multi_turn %} Consider the conversation as a whole, not just the last turn, unless a label description says otherwise.{% endif %}{% if allow_none %} Pick `NONE` only when no other label fits.{% else %} Pick the closest label even when the fit is imperfect.{% endif %}

deepeval-4.2.4/deepeval/classifiers/classifier/templates/_experimental_system_one_reason.txt

@@ -0,0 +1,39 @@
+You are a classifier. A calibrated judge has already assigned a label to the {% if turns %}conversation{% else %}test case{% endif %} below from the closed set of labels provided. Write a brief explanation for that label and return a JSON object with one field, `"reason"`.
+
+Your explanation should:
+- Be specific and grounded in the label descriptions, referencing details from the {% if turns %}conversation{% else %}test case{% endif %}.
+- Reflect the judge's probabilities: mention the labels it was most and least confident about when that helps.
+- Be concise, clear, and focused on why the chosen label fits better than the alternatives.
+- Not quote the numeric probabilities or confidence themselves.
+- Not invent labels that are not in the list, and not dispute the judge's label.
+{% if allow_none %}- If the label is `NONE`, explain why none of the listed labels apply.
+{% endif %}
+Only return valid JSON. Do **not** include any extra commentary or text.
+
+---
+
+Labels:
+{{ labels }}
+
+Judge's Label: {{ label }}
+
+Judge's Probabilities:
+{{ probabilities }}
+
+Judge's Confidence: {{ confidence }}
+{% if turns %}{% if test_case_content %}
+{{ test_case_content }}
+{% endif %}
+Turns:
+{{ turns }}
+{% else %}
+Test Case:
+{{ test_case_content }}
+{% endif %}
+---
+**Example JSON:**
+{
+  "reason": "your concise and informative reason here"
+}
+
+JSON:

deepeval-4.2.4/deepeval/classifiers/classifier/templates/class.txt

@@ -0,0 +1 @@
+Classifier

deepeval-4.2.4/deepeval/classifiers/classifier/templates/classify_multi_turn.txt

@@ -0,0 +1,31 @@
+You are a classifier. Given the conversation below, assign it exactly ONE label from the closed set of labels provided and return a JSON object with two fields:
+
+- `"label"`: the name of the single best-fitting label, copied exactly as written in the list of labels.{% if allow_none %} If none of the labels apply, return `"NONE"`.{% endif %}
+- `"reason"`: a brief explanation for why this label was chosen. Reference specific turns or details from the conversation. Do **not** invent labels that are not in the list.
+
+Rules:
+- Use the label descriptions as the boundary for each label. When a description is given, it is authoritative.
+- Consider the conversation as a whole, not just the last turn, unless a label description says otherwise.
+- Choose exactly one label. Never return multiple labels.
+{% if allow_none %}- Only return `"NONE"` when no label fits; do not use it to avoid a hard decision.
+{% else %}- You must choose one of the listed labels, even when the fit is imperfect. Pick the closest one and explain the reservation in the reason.
+{% endif %}- Only return valid JSON. Do **not** include any extra commentary or text.
+
+---
+
+Labels:
+{{ labels }}
+{% if test_case_content %}
+{{ test_case_content }}
+{% endif %}
+Turns:
+{{ turns }}
+
+---
+**Example JSON:**
+{
+  "label": "one of the label names above{% if allow_none %}, or NONE{% endif %}",
+  "reason": "your concise and informative reason here"
+}
+
+JSON: