deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/topic_adherence/topic_adherence.py

@@ -0,0 +1,355 @@
+from typing import Optional, List, Union
+
+from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    get_unit_interactions,
+    check_conversational_test_case_params,
+    initialize_model,
+)
+from deepeval.test_case import ConversationalTestCase, TurnParams
+from deepeval.metrics import BaseConversationalMetric
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.topic_adherence.template import TopicAdherenceTemplate
+from deepeval.metrics.topic_adherence.schema import (
+    RelevancyVerdict,
+    QAPairs,
+    QAPair,
+)
+from deepeval.metrics.api import metric_data_manager
+
+
+class TopicAdherenceMetric(BaseConversationalMetric):
+
+    _required_test_case_params = [
+        TurnParams.ROLE,
+        TurnParams.CONTENT,
+    ]
+
+    def __init__(
+        self,
+        relevant_topics: List[str],
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.relevant_topics = relevant_topics
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case, self._required_test_case_params, self
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                unit_interactions = get_unit_interactions(test_case.turns)
+                interaction_pairs = self._get_qa_pairs(unit_interactions)
+                True_Positives = [0, []]
+                True_Negatives = [0, []]
+                False_Positives = [0, []]
+                False_Negatives = [0, []]
+                for interaction_pair in interaction_pairs:
+                    for qa_pair in interaction_pair.qa_pairs:
+                        qa_verdict: RelevancyVerdict = self._get_qa_verdict(
+                            qa_pair
+                        )
+                        if qa_verdict.verdict == "TP":
+                            True_Positives[0] += 1
+                            True_Positives[1].append(qa_verdict.reason)
+                        elif qa_verdict.verdict == "TN":
+                            True_Negatives[0] += 1
+                            True_Negatives[1].append(qa_verdict.reason)
+                        elif qa_verdict.verdict == "FP":
+                            False_Positives[0] += 1
+                            False_Positives[1].append(qa_verdict.reason)
+                        elif qa_verdict.verdict == "FN":
+                            False_Negatives[0] += 1
+                            False_Negatives[1].append(qa_verdict.reason)
+
+                self.score = self._get_score(
+                    True_Positives,
+                    True_Negatives,
+                    False_Positives,
+                    False_Negatives,
+                )
+                self.success = self.score >= self.threshold
+                self.reason = self._generate_reason(
+                    True_Positives,
+                    True_Negatives,
+                    False_Positives,
+                    False_Negatives,
+                )
+
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Interaction Pairs: \n{prettify_list(interaction_pairs)} \n",
+                        f"Truth Table:",
+                        f"\nTrue Positives:",
+                        f"Count: {True_Positives[0]}, Reasons: {prettify_list(True_Positives[1])} \n",
+                        f"\nTrue Negatives: ",
+                        f"Count: {True_Negatives[0]}, Reasons: {prettify_list(True_Negatives[1])} \n",
+                        f"\nFalse Positives: ",
+                        f"Count: {False_Positives[0]}, Reasons: {prettify_list(False_Positives[1])} \n",
+                        f"\nFalse Negatives: ",
+                        f"Count: {False_Negatives[0]}, Reasons: {prettify_list(False_Negatives[1])} \n",
+                        f"Final Score: {self.score}",
+                        f"Final Reason: {self.reason}",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case, self._required_test_case_params, self
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            unit_interactions = get_unit_interactions(test_case.turns)
+            interaction_pairs = await self._a_get_qa_pairs(unit_interactions)
+            True_Positives = [0, []]
+            True_Negatives = [0, []]
+            False_Positives = [0, []]
+            False_Negatives = [0, []]
+            for interaction_pair in interaction_pairs:
+                for qa_pair in interaction_pair.qa_pairs:
+                    qa_verdict: RelevancyVerdict = self._get_qa_verdict(qa_pair)
+                    if qa_verdict.verdict == "TP":
+                        True_Positives[0] += 1
+                        True_Positives[1].append(qa_verdict.reason)
+                    elif qa_verdict.verdict == "TN":
+                        True_Negatives[0] += 1
+                        True_Negatives[1].append(qa_verdict.reason)
+                    elif qa_verdict.verdict == "FP":
+                        False_Positives[0] += 1
+                        False_Positives[1].append(qa_verdict.reason)
+                    elif qa_verdict.verdict == "FN":
+                        False_Negatives[0] += 1
+                        False_Negatives[1].append(qa_verdict.reason)
+
+            self.score = self._get_score(
+                True_Positives, True_Negatives, False_Positives, False_Negatives
+            )
+            self.success = self.score >= self.threshold
+            self.reason = await self._a_generate_reason(
+                True_Positives, True_Negatives, False_Positives, False_Negatives
+            )
+
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"Interaction Pairs: \n{prettify_list(interaction_pairs)} \n",
+                    f"Truth Table:",
+                    f"\nTrue Positives:",
+                    f"Count: {True_Positives[0]}, Reasons: {prettify_list(True_Positives[1])} \n",
+                    f"\nTrue Negatives: ",
+                    f"Count: {True_Negatives[0]}, Reasons: {prettify_list(True_Negatives[1])} \n",
+                    f"\nFalse Positives: ",
+                    f"Count: {False_Positives[0]}, Reasons: {prettify_list(False_Positives[1])} \n",
+                    f"\nFalse Negatives: ",
+                    f"Count: {False_Negatives[0]}, Reasons: {prettify_list(False_Negatives[1])} \n",
+                    f"Final Score: {self.score}",
+                    f"Final Reason: {self.reason}",
+                ],
+            )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    def _generate_reason(self, TP, TN, FP, FN):
+        total = TP[0] + TN[0] + FP[0] + FN[0]
+        if total <= 0:
+            return "There were no question-answer pairs to evaluate. Please enable verbose logs to look at the evaluation steps taken"
+        prompt = TopicAdherenceTemplate.generate_reason(
+            self.success, self.score, self.threshold, TP, TN, FP, FN
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = self.model.generate(prompt)
+            return res
+
+    async def _a_generate_reason(self, TP, TN, FP, FN):
+        prompt = TopicAdherenceTemplate.generate_reason(
+            self.success, self.score, self.threshold, TP, TN, FP, FN
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = await self.model.a_generate(prompt)
+            return res
+
+    def _get_score(self, TP, TN, FP, FN) -> float:
+        true_values = TP[0] + TN[0]
+        total = TP[0] + TN[0] + FP[0] + FN[0]
+        if total <= 0:
+            score = 0
+        else:
+            score = true_values / total
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    def _get_qa_verdict(self, qa_pair: QAPair) -> RelevancyVerdict:
+        prompt = TopicAdherenceTemplate.get_qa_pair_verdict(
+            self.relevant_topics, qa_pair.question, qa_pair.response
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=RelevancyVerdict)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = self.model.generate(prompt, schema=RelevancyVerdict)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return RelevancyVerdict(**data)
+
+    async def _a_get_qa_verdict(self, qa_pair: QAPair) -> RelevancyVerdict:
+        prompt = TopicAdherenceTemplate.get_qa_pair_verdict(
+            self.relevant_topics, qa_pair.question, qa_pair.response
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=RelevancyVerdict
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = await self.model.a_generate(
+                    prompt, schema=RelevancyVerdict
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return RelevancyVerdict(**data)
+
+    def _get_qa_pairs(self, unit_interactions: List) -> List[QAPairs]:
+        qa_pairs = []
+        for unit_interaction in unit_interactions:
+            conversation = "Conversation: \n"
+            for turn in unit_interaction:
+                conversation += f"{turn.role} \n"
+                conversation += f"{turn.content} \n\n"
+            prompt = TopicAdherenceTemplate.get_qa_pairs(conversation)
+            new_pair = None
+
+            if self.using_native_model:
+                res, cost = self.model.generate(prompt, schema=QAPairs)
+                self.evaluation_cost += cost
+                new_pair = res
+            else:
+                try:
+                    res = self.model.generate(prompt, schema=QAPairs)
+                    new_pair = res
+                except TypeError:
+                    res = self.model.generate(prompt)
+                    data = trimAndLoadJson(res, self)
+                    new_pair = QAPairs(**data)
+
+            if new_pair is not None:
+                qa_pairs.append(new_pair)
+
+        return qa_pairs
+
+    async def _a_get_qa_pairs(self, unit_interactions: List) -> List[QAPairs]:
+        qa_pairs = []
+        for unit_interaction in unit_interactions:
+            conversation = "Conversation: \n"
+            for turn in unit_interaction:
+                conversation += f"{turn.role} \n"
+                conversation += f"{turn.content} \n\n"
+            prompt = TopicAdherenceTemplate.get_qa_pairs(conversation)
+            new_pair = None
+
+            if self.using_native_model:
+                res, cost = await self.model.a_generate(prompt, schema=QAPairs)
+                self.evaluation_cost += cost
+                new_pair = res
+            else:
+                try:
+                    res = await self.model.a_generate(prompt, schema=QAPairs)
+                    new_pair = res
+                except TypeError:
+                    res = await self.model.a_generate(prompt)
+                    data = trimAndLoadJson(res, self)
+                    new_pair = QAPairs(**data)
+
+            if new_pair is not None:
+                qa_pairs.append(new_pair)
+
+        return qa_pairs
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Topic Adherence"

deepeval/metrics/toxicity/toxicity.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.toxicity.template import ToxicityTemplate
 from deepeval.metrics.toxicity.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ToxicityMetric(BaseMetric):
@@ -50,6 +51,7 @@ class ToxicityMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -65,6 +67,7 @@ class ToxicityMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class ToxicityMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -92,6 +99,7 @@ class ToxicityMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -122,6 +130,10 @@ class ToxicityMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/turn_relevancy/turn_relevancy.py

@@ -20,6 +20,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, Turn, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.turn_relevancy.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class TurnRelevancyMetric(BaseConversationalMetric):
@@ -49,6 +50,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -65,6 +67,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -91,6 +94,10 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -98,6 +105,7 @@ class TurnRelevancyMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -134,6 +142,10 @@ class TurnRelevancyMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/models/embedding_models/azure_embedding_model.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict, List, Optional
 from openai import AzureOpenAI, AsyncAzureOpenAI
 from deepeval.key_handler import (
     EmbeddingKeyValues,
@@ -17,28 +17,39 @@ retry_azure = create_retry_decorator(PS.AZURE)
 
 
 class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
-    def __init__(self, **kwargs):
-        self.azure_openai_api_key = KEY_FILE_HANDLER.fetch_data(
+    def __init__(
+        self,
+        openai_api_key: Optional[str] = None,
+        openai_api_version: Optional[str] = None,
+        azure_endpoint: Optional[str] = None,
+        azure_deployment: Optional[str] = None,
+        model: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **client_kwargs,
+    ):
+        self.openai_api_key = openai_api_key or KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.AZURE_OPENAI_API_KEY
         )
-        self.openai_api_version = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.OPENAI_API_VERSION
+        self.openai_api_version = (
+            openai_api_version
+            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.OPENAI_API_VERSION)
         )
-        self.azure_embedding_deployment = KEY_FILE_HANDLER.fetch_data(
-            EmbeddingKeyValues.AZURE_EMBEDDING_DEPLOYMENT_NAME
-        )
-        self.azure_endpoint = KEY_FILE_HANDLER.fetch_data(
+        self.azure_endpoint = azure_endpoint or KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.AZURE_OPENAI_ENDPOINT
         )
-        self.model_name = self.azure_embedding_deployment
-        self.kwargs = kwargs
+        self.azure_deployment = azure_deployment or KEY_FILE_HANDLER.fetch_data(
+            EmbeddingKeyValues.AZURE_EMBEDDING_DEPLOYMENT_NAME
+        )
+        self.client_kwargs = client_kwargs or {}
+        self.model_name = model or self.azure_deployment
+        self.generation_kwargs = generation_kwargs or {}
+        super().__init__(self.model_name)
 
     @retry_azure
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=text,
-            model=self.azure_embedding_deployment,
+            input=text, model=self.model_name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -46,8 +57,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
-            input=texts,
-            model=self.azure_embedding_deployment,
+            input=texts, model=self.model_name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -55,8 +65,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=text,
-            model=self.azure_embedding_deployment,
+            input=text, model=self.model_name, **self.generation_kwargs
         )
         return response.data[0].embedding
 
@@ -64,8 +73,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
-            input=texts,
-            model=self.azure_embedding_deployment,
+            input=texts, model=self.model_name, **self.generation_kwargs
         )
         return [item.embedding for item in response.data]
 
@@ -77,30 +85,23 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
             return self._build_client(AzureOpenAI)
         return self._build_client(AsyncAzureOpenAI)
 
-    def _client_kwargs(self) -> Dict:
-        """
-        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
-        If the user opts into SDK retries for 'azure' via DEEPEVAL_SDK_RETRY_PROVIDERS,
-        leave their retry settings as is.
-        """
-        kwargs = dict(self.kwargs or {})
+    def _build_client(self, cls):
+        client_kwargs = self.client_kwargs.copy()
         if not sdk_retries_for(PS.AZURE):
-            kwargs["max_retries"] = 0
-        return kwargs
+            client_kwargs["max_retries"] = 0
 
-    def _build_client(self, cls):
-        kw = dict(
-            api_key=self.azure_openai_api_key,
+        client_init_kwargs = dict(
+            api_key=self.openai_api_key,
             api_version=self.openai_api_version,
             azure_endpoint=self.azure_endpoint,
-            azure_deployment=self.azure_embedding_deployment,
-            **self._client_kwargs(),
+            azure_deployment=self.azure_deployment,
+            **client_kwargs,
         )
         try:
-            return cls(**kw)
+            return cls(**client_init_kwargs)
         except TypeError as e:
             # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
             if "max_retries" in str(e):
-                kw.pop("max_retries", None)
-                return cls(**kw)
+                client_init_kwargs.pop("max_retries", None)
+                return cls(**client_init_kwargs)
             raise