deepeval 3.7.6__py3-none-any.whl → 3.7.8__py3-none-any.whl

deepeval/metrics/topic_adherence/topic_adherence.py

@@ -18,6 +18,7 @@ from deepeval.metrics.topic_adherence.schema import (
     RelevancyVerdict,
     QAPairs,
     QAPair,
+    TopicAdherenceReason,
 )
 from deepeval.metrics.api import metric_data_manager
 
@@ -227,25 +228,25 @@ class TopicAdherenceMetric(BaseConversationalMetric):
         prompt = TopicAdherenceTemplate.generate_reason(
             self.success, self.score, self.threshold, TP, TN, FP, FN
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TopicAdherenceReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(self, TP, TN, FP, FN):
         prompt = TopicAdherenceTemplate.generate_reason(
             self.success, self.score, self.threshold, TP, TN, FP, FN
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TopicAdherenceReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_score(self, TP, TN, FP, FN) -> float:
         true_values = TP[0] + TN[0]

deepeval/metrics/turn_contextual_precision/template.py

@@ -134,6 +134,13 @@ class TurnContextualPrecisionTemplate:
             Context:
             This metric evaluates conversational contextual precision by determining whether relevant nodes in retrieval context are ranked higher than irrelevant nodes for each interaction. Each interaction yields a reason indicating why relevant nodes were well-ranked or poorly-ranked. You are given all those reasons.
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_precision_score> because <your_reason>."
+            }}
+
             Inputs:
             - final_score: the averaged score across all interactions.
             - success: whether the metric passed or failed
@@ -160,7 +167,7 @@ class TurnContextualPrecisionTemplate:
 
             Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
 
-            The final reason:
+            JSON:
             """
         )
 

deepeval/metrics/turn_contextual_precision/turn_contextual_precision.py

@@ -14,6 +14,8 @@ from deepeval.metrics.utils import (
     get_unit_interactions,
     get_turns_in_sliding_window,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.turn_contextual_precision.template import (
@@ -279,26 +281,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     def _generate_verdicts(
         self,
@@ -319,24 +308,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     async def _a_get_interaction_score_and_reason(
         self,
@@ -438,24 +416,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualPrecisionScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualPrecisionScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=ContextualPrecisionScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_interaction_reason(
         self,
@@ -485,22 +452,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualPrecisionScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualPrecisionScoreReason = self.model.generate(
-                    prompt, schema=ContextualPrecisionScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_verbose_steps(
         self, interaction_scores: List[InteractionContextualPrecisionScore]
@@ -533,13 +491,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self, scores: List[InteractionContextualPrecisionScore]
@@ -558,13 +516,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(
         self, scores: List[InteractionContextualPrecisionScore]

deepeval/metrics/turn_contextual_recall/template.py

@@ -125,6 +125,13 @@ class TurnContextualRecallTemplate:
             Context:
             This metric evaluates conversational contextual recall by determining whether sentences in the assistant output can be attributed to the retrieval context for each interaction. Each interaction yields a reason indicating which sentences were supported or unsupported. You are given all those reasons.
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_recall_score> because <your_reason>."
+            }}
+
             Inputs:
             - final_score: the averaged score across all interactions.
             - success: whether the metric passed or failed
@@ -151,7 +158,7 @@ class TurnContextualRecallTemplate:
 
             Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
 
-            The final reason:
+            JSON:
             """
         )
 

deepeval/metrics/turn_contextual_recall/turn_contextual_recall.py

@@ -14,6 +14,8 @@ from deepeval.metrics.utils import (
     get_unit_interactions,
     get_turns_in_sliding_window,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.turn_contextual_recall.template import (
@@ -271,25 +273,13 @@ class TurnContextualRecallMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     def _generate_verdicts(
         self,
@@ -308,23 +298,13 @@ class TurnContextualRecallMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     async def _a_get_interaction_score_and_reason(
         self,
@@ -412,22 +392,13 @@ class TurnContextualRecallMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRecallScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRecallScoreReason = await self.model.a_generate(
-                    prompt, schema=ContextualRecallScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRecallScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_interaction_reason(
         self,
@@ -456,22 +427,13 @@ class TurnContextualRecallMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRecallScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRecallScoreReason = self.model.generate(
-                    prompt, schema=ContextualRecallScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRecallScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_verbose_steps(
         self, interaction_scores: List[InteractionContextualRecallScore]
@@ -504,13 +466,13 @@ class TurnContextualRecallMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRecallScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self, scores: List[InteractionContextualRecallScore]
@@ -529,13 +491,13 @@ class TurnContextualRecallMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRecallScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(
         self, scores: List[InteractionContextualRecallScore]

deepeval/metrics/turn_contextual_relevancy/template.py

@@ -130,6 +130,13 @@ class TurnContextualRelevancyTemplate:
             Context:
             This metric evaluates conversational contextual relevancy by determining whether statements in the retrieval context are relevant to the user message for each interaction. Each interaction yields a reason indicating which statements were relevant or irrelevant. You are given all those reasons.
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_relevancy_score> because <your_reason>."
+            }}
+
             Inputs:
             - final_score: the averaged score across all interactions.
             - success: whether the metric passed or failed
@@ -156,6 +163,6 @@ class TurnContextualRelevancyTemplate:
 
             Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
 
-            The final reason:
+            JSON:
             """
         )

deepeval/metrics/turn_contextual_relevancy/turn_contextual_relevancy.py

@@ -14,6 +14,8 @@ from deepeval.metrics.utils import (
     get_unit_interactions,
     get_turns_in_sliding_window,
     initialize_model,
+    generate_with_schema_and_extract,
+    a_generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.turn_contextual_relevancy.template import (
@@ -264,29 +266,15 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
                 multimodal=multimodal,
             )
 
-            if self.using_native_model:
-                res, cost = await self.model.a_generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                self.evaluation_cost += cost
-                verdicts.extend([item for item in res.verdicts])
-            else:
-                try:
-                    res: ContextualRelevancyVerdicts = (
-                        await self.model.a_generate(
-                            prompt, schema=ContextualRelevancyVerdicts
-                        )
-                    )
-                    verdicts.extend([item for item in res.verdicts])
-                except TypeError:
-                    res = await self.model.a_generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    verdicts.extend(
-                        [
-                            ContextualRelevancyVerdict(**item)
-                            for item in data["verdicts"]
-                        ]
-                    )
+            result = await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=ContextualRelevancyVerdicts,
+                extract_schema=lambda s: s.verdicts,
+                extract_json=lambda data: data["verdicts"],
+            )
+
+            verdicts.extend(result)
 
         return verdicts
 
@@ -306,27 +294,15 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
                 multimodal=multimodal,
             )
 
-            if self.using_native_model:
-                res, cost = self.model.generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                self.evaluation_cost += cost
-                verdicts.extend([item for item in res.verdicts])
-            else:
-                try:
-                    res: ContextualRelevancyVerdicts = self.model.generate(
-                        prompt, schema=ContextualRelevancyVerdicts
-                    )
-                    verdicts.extend([item for item in res.verdicts])
-                except TypeError:
-                    res = self.model.generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    verdicts.extend(
-                        [
-                            ContextualRelevancyVerdict(**item)
-                            for item in data["verdicts"]
-                        ]
-                    )
+            result = generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=ContextualRelevancyVerdicts,
+                extract_schema=lambda s: s.verdicts,
+                extract_json=lambda data: data["verdicts"],
+            )
+
+            verdicts.extend(result)
 
         return verdicts
 
@@ -419,24 +395,13 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRelevancyScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=ContextualRelevancyScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_interaction_reason(
         self,
@@ -469,22 +434,13 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRelevancyScoreReason = self.model.generate(
-                    prompt, schema=ContextualRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_verbose_steps(
         self, windows_scores: List[InteractionContextualRelevancyScore]
@@ -517,13 +473,13 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self, scores: List[InteractionContextualRelevancyScore]
@@ -542,13 +498,13 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(
         self, scores: List[InteractionContextualRelevancyScore]