deepeval 3.6.7__py3-none-any.whl → 3.6.9__py3-none-any.whl

deepeval/metrics/topic_adherence/schema.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+from typing import List, Dict, Literal
+
+
+class QAPair(BaseModel):
+    question: str
+    response: str
+
+
+class QAPairs(BaseModel):
+    qa_pairs: List[QAPair]
+
+
+class RelevancyVerdict(BaseModel):
+    verdict: Literal["TP", "TN", "FP", "FN"]
+    reason: str

deepeval/metrics/topic_adherence/template.py

@@ -0,0 +1,162 @@
+from typing import List
+import textwrap
+
+
+class TopicAdherenceTemplate:
+
+    @staticmethod
+    def get_qa_pairs(
+        conversation: str,
+    ) -> str:
+        return textwrap.dedent(
+            f"""Your task is to extract question-answer (QA) pairs from a multi-turn conversation between a `user` and an `assistant`.
+
+                You must return only valid pairs where:
+                - The **question** comes from the `user`.
+                - The **response** comes from the `assistant`.
+                - Both question and response must appear **explicitly** in the conversation.
+
+                Do not infer information beyond what is stated. Ignore irrelevant or conversational turns (e.g. greetings, affirmations) that do not constitute clear QA pairs.
+                If there are multiple questions and multiple answers in a single sentence, break them into separate pairs. Each pair must be standalone, and should not contain more than one question or response.
+
+                OUTPUT Format:
+                Return a **JSON object** with a single 2 keys:
+                - `"question"`: the user's question
+                - `"response"`: the assistant's direct response
+
+                If no valid QA pairs are found, return:
+                ```json
+                {{
+                    question: "",
+                    response: ""
+                }}
+
+                CHAIN OF THOUGHT:
+                - Read the full conversation sequentially.
+                - Identify user turns that clearly ask a question (explicit or strongly implied).
+                - Match each question with the immediate assistant response.
+                - Only include pairs where the assistant's reply directly addresses the user's question.
+                - Do not include incomplete, ambiguous, or out-of-context entries.
+
+                EXAMPLE:
+                    
+                Conversation:
+
+                user: Which food is best for diabetic patients?
+                assistant: Steel-cut oats are good for diabetic patients
+                user: Is it better if I eat muesli instead of oats?
+                assistant: While muesli is good for diabetic people, steel-cut oats are preferred. Refer to your nutritionist for better guidance.
+
+                Example JSON:
+                {{
+                    "question": "Which food is best for diabetic patients?",
+                    "response": "Steel-cut oats are good for diabetic patients"
+                }}
+                ===== END OF EXAMPLE ======
+
+                **
+                IMPORTANT: Please make sure to only return in JSON format with one key: 'qa_pairs' and the value MUST be a list of dictionaries
+                **
+
+                Conversation: 
+                {conversation}
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def get_qa_pair_verdict(
+        relevant_topics: List[str],
+        question: str,
+        response: str,
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are given:
+                - A list of **relevant topics**
+                - A **user question**
+                - An **assistant response**
+
+                Your task is to:
+                1. Determine if the question is relevant to the list of topics.
+                2. If it is relevant, evaluate whether the response properly answers the question.
+                3. Based on both relevance and correctness, assign one of four possible verdicts.
+                4. Give a simple, comprehensive reason explaining why this question-answer pair was assigned this verdict
+
+                VERDICTS:
+                - `"TP"` (True Positive): Question is relevant and the response correctly answers it.
+                - `"FN"` (False Negative): Question is relevant, but the assistant refused to answer or gave an irrelevant response.
+                - `"FP"` (False Positive): Question is NOT relevant, but the assistant still gave an answer (based on general/training knowledge).
+                - `"TN"` (True Negative): Question is NOT relevant, and the assistant correctly refused to answer.
+
+                OUTPUT FORMAT:
+                Return only a **JSON object** with one key:
+                ```json
+                {{
+                    "verdict": "TP"  // or TN, FP, FN
+                    "reason": "Reason why the verdict is 'TP'"
+                }}
+
+                CHAIN OF THOUGHT:
+                - Check if the question aligns with any of the relevant topics.
+                - If yes:
+                    - Assess if the response is correct, complete, and directly answers the question.
+                - If no:
+                    - Check if the assistant refused appropriately or gave an unwarranted answer.
+                - Choose the correct verdict using the definitions above.
+
+                EXAMPLE:
+
+                Relevant topics: ["heath nutrition", "food and their benefits"]
+                Question: "Which food is best for diabetic patients?"
+                Response: "Steel-cut oats are good for diabetic patients"
+
+                Example JSON:
+                {{
+                    "verdict": "TP",
+                    "reason": The question asks about food for diabetic patients and the response clearly answers that oats are good for diabetic patients. Both align with the relevant topics of heath nutrition and food and their benefits... 
+                }}
+
+                ===== END OF EXAMPLE ======
+
+                **
+                IMPORTANT: Please make sure to only return in JSON format with two keys: 'verdict' and 'reason'
+                **
+
+                Relevant topics: {relevant_topics}
+                Question: {question}
+                Response: {response}
+
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def generate_reason(success, score, threshold, TP, TN, FP, FN) -> str:
+        return textwrap.dedent(
+            f"""You are given a score for a metric that calculates whether an agent has adhered to it's topics. 
+                You are also given a list of reasons for the truth table values that were used to calculate final score.
+                
+                Your task is to go through these reasons and give a single final explaination that clearly explains why this metric has failed or passed.
+
+                Pass: {success}
+                Score: {score}
+                Threshold: {threshold}
+
+                Here are the reasons for all truth table entries:
+
+                True positive reasons: {TP[1]}
+                True negative reasons: {TN[1]}
+                False positives reasons: {FP[1]}
+                False negatives reasons: {FN[1]}
+
+                Score calculation = Number of True Positives + Number of True Negatives / Total number of table entries
+
+                **
+                IMPORTANT: Now generate a comprehensive reason that explains why this metric failed. You MUST output only the reason as a string and nothing else.
+                **
+
+                Output ONLY the reason, DON"T output anything else.
+
+                Reason:
+            """
+        )

deepeval/metrics/topic_adherence/topic_adherence.py

@@ -0,0 +1,355 @@
+from typing import Optional, List, Union
+
+from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    get_unit_interactions,
+    check_conversational_test_case_params,
+    initialize_model,
+)
+from deepeval.test_case import ConversationalTestCase, TurnParams
+from deepeval.metrics import BaseConversationalMetric
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.topic_adherence.template import TopicAdherenceTemplate
+from deepeval.metrics.topic_adherence.schema import (
+    RelevancyVerdict,
+    QAPairs,
+    QAPair,
+)
+from deepeval.metrics.api import metric_data_manager
+
+
+class TopicAdherenceMetric(BaseConversationalMetric):
+
+    _required_test_case_params = [
+        TurnParams.ROLE,
+        TurnParams.CONTENT,
+    ]
+
+    def __init__(
+        self,
+        relevant_topics: List[str],
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.relevant_topics = relevant_topics
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case, self._required_test_case_params, self
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                unit_interactions = get_unit_interactions(test_case.turns)
+                interaction_pairs = self._get_qa_pairs(unit_interactions)
+                True_Positives = [0, []]
+                True_Negatives = [0, []]
+                False_Positives = [0, []]
+                False_Negatives = [0, []]
+                for interaction_pair in interaction_pairs:
+                    for qa_pair in interaction_pair.qa_pairs:
+                        qa_verdict: RelevancyVerdict = self._get_qa_verdict(
+                            qa_pair
+                        )
+                        if qa_verdict.verdict == "TP":
+                            True_Positives[0] += 1
+                            True_Positives[1].append(qa_verdict.reason)
+                        elif qa_verdict.verdict == "TN":
+                            True_Negatives[0] += 1
+                            True_Negatives[1].append(qa_verdict.reason)
+                        elif qa_verdict.verdict == "FP":
+                            False_Positives[0] += 1
+                            False_Positives[1].append(qa_verdict.reason)
+                        elif qa_verdict.verdict == "FN":
+                            False_Negatives[0] += 1
+                            False_Negatives[1].append(qa_verdict.reason)
+
+                self.score = self._get_score(
+                    True_Positives,
+                    True_Negatives,
+                    False_Positives,
+                    False_Negatives,
+                )
+                self.success = self.score >= self.threshold
+                self.reason = self._generate_reason(
+                    True_Positives,
+                    True_Negatives,
+                    False_Positives,
+                    False_Negatives,
+                )
+
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Interaction Pairs: \n{prettify_list(interaction_pairs)} \n",
+                        f"Truth Table:",
+                        f"\nTrue Positives:",
+                        f"Count: {True_Positives[0]}, Reasons: {prettify_list(True_Positives[1])} \n",
+                        f"\nTrue Negatives: ",
+                        f"Count: {True_Negatives[0]}, Reasons: {prettify_list(True_Negatives[1])} \n",
+                        f"\nFalse Positives: ",
+                        f"Count: {False_Positives[0]}, Reasons: {prettify_list(False_Positives[1])} \n",
+                        f"\nFalse Negatives: ",
+                        f"Count: {False_Negatives[0]}, Reasons: {prettify_list(False_Negatives[1])} \n",
+                        f"Final Score: {self.score}",
+                        f"Final Reason: {self.reason}",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case, self._required_test_case_params, self
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            unit_interactions = get_unit_interactions(test_case.turns)
+            interaction_pairs = await self._a_get_qa_pairs(unit_interactions)
+            True_Positives = [0, []]
+            True_Negatives = [0, []]
+            False_Positives = [0, []]
+            False_Negatives = [0, []]
+            for interaction_pair in interaction_pairs:
+                for qa_pair in interaction_pair.qa_pairs:
+                    qa_verdict: RelevancyVerdict = self._get_qa_verdict(qa_pair)
+                    if qa_verdict.verdict == "TP":
+                        True_Positives[0] += 1
+                        True_Positives[1].append(qa_verdict.reason)
+                    elif qa_verdict.verdict == "TN":
+                        True_Negatives[0] += 1
+                        True_Negatives[1].append(qa_verdict.reason)
+                    elif qa_verdict.verdict == "FP":
+                        False_Positives[0] += 1
+                        False_Positives[1].append(qa_verdict.reason)
+                    elif qa_verdict.verdict == "FN":
+                        False_Negatives[0] += 1
+                        False_Negatives[1].append(qa_verdict.reason)
+
+            self.score = self._get_score(
+                True_Positives, True_Negatives, False_Positives, False_Negatives
+            )
+            self.success = self.score >= self.threshold
+            self.reason = await self._a_generate_reason(
+                True_Positives, True_Negatives, False_Positives, False_Negatives
+            )
+
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"Interaction Pairs: \n{prettify_list(interaction_pairs)} \n",
+                    f"Truth Table:",
+                    f"\nTrue Positives:",
+                    f"Count: {True_Positives[0]}, Reasons: {prettify_list(True_Positives[1])} \n",
+                    f"\nTrue Negatives: ",
+                    f"Count: {True_Negatives[0]}, Reasons: {prettify_list(True_Negatives[1])} \n",
+                    f"\nFalse Positives: ",
+                    f"Count: {False_Positives[0]}, Reasons: {prettify_list(False_Positives[1])} \n",
+                    f"\nFalse Negatives: ",
+                    f"Count: {False_Negatives[0]}, Reasons: {prettify_list(False_Negatives[1])} \n",
+                    f"Final Score: {self.score}",
+                    f"Final Reason: {self.reason}",
+                ],
+            )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    def _generate_reason(self, TP, TN, FP, FN):
+        total = TP[0] + TN[0] + FP[0] + FN[0]
+        if total <= 0:
+            return "There were no question-answer pairs to evaluate. Please enable verbose logs to look at the evaluation steps taken"
+        prompt = TopicAdherenceTemplate.generate_reason(
+            self.success, self.score, self.threshold, TP, TN, FP, FN
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = self.model.generate(prompt)
+            return res
+
+    async def _a_generate_reason(self, TP, TN, FP, FN):
+        prompt = TopicAdherenceTemplate.generate_reason(
+            self.success, self.score, self.threshold, TP, TN, FP, FN
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = await self.model.a_generate(prompt)
+            return res
+
+    def _get_score(self, TP, TN, FP, FN) -> float:
+        true_values = TP[0] + TN[0]
+        total = TP[0] + TN[0] + FP[0] + FN[0]
+        if total <= 0:
+            score = 0
+        else:
+            score = true_values / total
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    def _get_qa_verdict(self, qa_pair: QAPair) -> RelevancyVerdict:
+        prompt = TopicAdherenceTemplate.get_qa_pair_verdict(
+            self.relevant_topics, qa_pair.question, qa_pair.response
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=RelevancyVerdict)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = self.model.generate(prompt, schema=RelevancyVerdict)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return RelevancyVerdict(**data)
+
+    async def _a_get_qa_verdict(self, qa_pair: QAPair) -> RelevancyVerdict:
+        prompt = TopicAdherenceTemplate.get_qa_pair_verdict(
+            self.relevant_topics, qa_pair.question, qa_pair.response
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=RelevancyVerdict
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = await self.model.a_generate(
+                    prompt, schema=RelevancyVerdict
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return RelevancyVerdict(**data)
+
+    def _get_qa_pairs(self, unit_interactions: List) -> List[QAPairs]:
+        qa_pairs = []
+        for unit_interaction in unit_interactions:
+            conversation = "Conversation: \n"
+            for turn in unit_interaction:
+                conversation += f"{turn.role} \n"
+                conversation += f"{turn.content} \n\n"
+            prompt = TopicAdherenceTemplate.get_qa_pairs(conversation)
+            new_pair = None
+
+            if self.using_native_model:
+                res, cost = self.model.generate(prompt, schema=QAPairs)
+                self.evaluation_cost += cost
+                new_pair = res
+            else:
+                try:
+                    res = self.model.generate(prompt, schema=QAPairs)
+                    new_pair = res
+                except TypeError:
+                    res = self.model.generate(prompt)
+                    data = trimAndLoadJson(res, self)
+                    new_pair = QAPairs(**data)
+
+            if new_pair is not None:
+                qa_pairs.append(new_pair)
+
+        return qa_pairs
+
+    async def _a_get_qa_pairs(self, unit_interactions: List) -> List[QAPairs]:
+        qa_pairs = []
+        for unit_interaction in unit_interactions:
+            conversation = "Conversation: \n"
+            for turn in unit_interaction:
+                conversation += f"{turn.role} \n"
+                conversation += f"{turn.content} \n\n"
+            prompt = TopicAdherenceTemplate.get_qa_pairs(conversation)
+            new_pair = None
+
+            if self.using_native_model:
+                res, cost = await self.model.a_generate(prompt, schema=QAPairs)
+                self.evaluation_cost += cost
+                new_pair = res
+            else:
+                try:
+                    res = await self.model.a_generate(prompt, schema=QAPairs)
+                    new_pair = res
+                except TypeError:
+                    res = await self.model.a_generate(prompt)
+                    data = trimAndLoadJson(res, self)
+                    new_pair = QAPairs(**data)
+
+            if new_pair is not None:
+                qa_pairs.append(new_pair)
+
+        return qa_pairs
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Topic Adherence"

deepeval/models/embedding_models/azure_embedding_model.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict, List, Optional
 from openai import AzureOpenAI, AsyncAzureOpenAI
 from deepeval.key_handler import (
     EmbeddingKeyValues,
@@ -17,28 +17,39 @@ retry_azure = create_retry_decorator(PS.AZURE)
 
 
 class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
-    def __init__(self, **kwargs):
-        self.azure_openai_api_key = KEY_FILE_HANDLER.fetch_data(
+    def __init__(
+        self,
+        openai_api_key: Optional[str] = None,
+        openai_api_version: Optional[str] = None,
+        azure_endpoint: Optional[str] = None,
+        azure_deployment: Optional[str] = None,
+        model: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **client_kwargs,
+    ):
+        self.openai_api_key = openai_api_key or KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.AZURE_OPENAI_API_KEY
         )
-        self.openai_api_version = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.OPENAI_API_VERSION
+        self.openai_api_version = (
+            openai_api_version
+            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.OPENAI_API_VERSION)
         )
-        self.azure_embedding_deployment = KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.AZURE_EMBEDDING_DEPLOYMENT_NAME
-        )
-        self.azure_endpoint = KEY_FILE_HANDLER.fetch_data(
+        self.azure_endpoint = azure_endpoint or KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.AZURE_OPENAI_ENDPOINT
         )
-        self.model_name = self.azure_embedding_deployment
-        self.kwargs = kwargs
+        self.azure_deployment = azure_deployment or KEY_FILE_HANDLER.fetch_data(
+            EmbeddingKeyValues.AZURE_EMBEDDING_DEPLOYMENT_NAME
+        )
+        self.client_kwargs = client_kwargs or {}
+        self.model_name = model or self.azure_deployment
+        self.generation_kwargs = generation_kwargs or {}
+        super().__init__(self.model_name)
 
     @retry_azure
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=text,
-            model=self.azure_embedding_deployment,
+            input=text, model=self.model_name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -46,8 +57,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=texts,
-            model=self.azure_embedding_deployment,
+            input=texts, model=self.model_name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -55,8 +65,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=text,
-            model=self.azure_embedding_deployment,
+            input=text, model=self.model_name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -64,8 +73,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=texts,
-            model=self.azure_embedding_deployment,
+            input=texts, model=self.model_name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -77,30 +85,23 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
             return self._build_client(AzureOpenAI)
         return self._build_client(AsyncAzureOpenAI)
 
-    def _client_kwargs(self) -> Dict:
-        """
-        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
-        If the user opts into SDK retries for 'azure' via DEEPEVAL_SDK_RETRY_PROVIDERS,
-        leave their retry settings as is.
-        """
-        kwargs = dict(self.kwargs or {})
+    def _build_client(self, cls):
+        client_kwargs = self.client_kwargs.copy()
         if not sdk_retries_for(PS.AZURE):
-            kwargs["max_retries"] = 0
-        return kwargs
+            client_kwargs["max_retries"] = 0
 
-    def _build_client(self, cls):
-        kw = dict(
-            api_key=self.azure_openai_api_key,
+        client_init_kwargs = dict(
+            api_key=self.openai_api_key,
             api_version=self.openai_api_version,
             azure_endpoint=self.azure_endpoint,
-            azure_deployment=self.azure_embedding_deployment,
-            **self._client_kwargs(),
+            azure_deployment=self.azure_deployment,
+            **client_kwargs,
         )
         try:
-            return cls(**kw)
+            return cls(**client_init_kwargs)
         except TypeError as e:
             # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
             if "max_retries" in str(e):
-                kw.pop("max_retries", None)
-                return cls(**kw)
+                client_init_kwargs.pop("max_retries", None)
+                return cls(**client_init_kwargs)
             raise