deepeval 3.7.6__py3-none-any.whl → 3.7.8__py3-none-any.whl

deepeval/metrics/turn_faithfulness/template.py

@@ -187,6 +187,13 @@ class TurnFaithfulnessTemplate:
                 Context:
                 This metric evaluates conversational faithfulness by extracting truths from retrieval context, extracting claims from the assistant's output, and generating verdicts that compare each claim against the truths. Each interaction yields a reason indicating why a verdict failed or succeeded. You are given all those reasons.
 
+                **
+                IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+                Example JSON:
+                {{
+                    "reason": "The score is <turn_faithfulness_score> because <your_reason>."
+                }}
+
                 Inputs:
                 - final_score: the averaged score across all interactions.
                 - success: whether the metric passed or failed
@@ -213,6 +220,6 @@ class TurnFaithfulnessTemplate:
 
                 Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
 
-                The final reason:
+                JSON:
             """
         )

deepeval/metrics/turn_faithfulness/turn_faithfulness.py

@@ -14,6 +14,8 @@ from deepeval.metrics.utils import (
     get_unit_interactions,
     get_turns_in_sliding_window,
     initialize_model,
+    generate_with_schema_and_extract,
+    a_generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.turn_faithfulness.template import (
@@ -273,18 +275,14 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             extraction_limit=self.truths_extraction_limit,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = await self.model.a_generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
+        )
 
     def _generate_truths(
         self, retrieval_context: str, multimodal: bool
@@ -294,18 +292,14 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             extraction_limit=self.truths_extraction_limit,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = self.model.generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
+        )
 
     async def _a_generate_claims(
         self, user_content: str, assistant_content: str, multimodal: bool
@@ -315,18 +309,14 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             assistant_output=assistant_content,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = await self.model.a_generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
+        )
 
     def _generate_claims(
         self, user_content: str, assistant_content: str, multimodal: bool
@@ -336,18 +326,14 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             assistant_output=assistant_content,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = self.model.generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
+        )
 
     async def _a_generate_verdicts(
         self, claims: Claims, truths: Truths, multimodal: bool
@@ -363,25 +349,13 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     def _generate_verdicts(
         self, claims: Claims, truths: Truths, multimodal: bool
@@ -397,23 +371,13 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     def _get_interaction_score_and_reason(
         self, verdicts, multimodal: bool
@@ -486,22 +450,13 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=FaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: FaithfulnessScoreReason = await self.model.a_generate(
-                    prompt, schema=FaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_interaction_reason(self, score, verdicts, multimodal: bool) -> str:
         if self.include_reason is False:
@@ -518,22 +473,13 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=FaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: FaithfulnessScoreReason = self.model.generate(
-                    prompt, schema=FaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_verbose_steps(
         self, interaction_scores: List[InteractionFaithfulnessScore]
@@ -568,13 +514,13 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self, scores: List[InteractionFaithfulnessScore]
@@ -593,13 +539,13 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(
         self, scores: List[InteractionFaithfulnessScore]

deepeval/metrics/utils.py

@@ -32,6 +32,7 @@ from deepeval.models import (
     GeminiModel,
     AmazonBedrockModel,
     LiteLLMModel,
+    PortkeyModel,
     KimiModel,
     GrokModel,
     DeepSeekModel,
@@ -458,6 +459,11 @@ async def a_generate_with_schema_and_extract(
 ###############################################
 
 
+def should_use_anthropic_model():
+    value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_ANTHROPIC_MODEL)
+    return value.lower() == "yes" if value is not None else False
+
+
 def should_use_azure_openai():
     value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_AZURE_OPENAI)
     return value.lower() == "yes" if value is not None else False
@@ -488,6 +494,11 @@ def should_use_litellm():
     return value.lower() == "yes" if value is not None else False
 
 
+def should_use_portkey():
+    value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_PORTKEY_MODEL)
+    return value.lower() == "yes" if value is not None else False
+
+
 def should_use_deepseek_model():
     value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.USE_DEEPSEEK_MODEL)
     return value.lower() == "yes" if value is not None else False
@@ -526,6 +537,8 @@ def initialize_model(
         return GeminiModel(), True
     if should_use_litellm():
         return LiteLLMModel(), True
+    if should_use_portkey():
+        return PortkeyModel(), True
     if should_use_ollama_model():
         return OllamaModel(), True
     elif should_use_local_model():
@@ -535,9 +548,11 @@ def initialize_model(
     elif should_use_moonshot_model():
         return KimiModel(model=model), True
     elif should_use_grok_model():
-        return GrokModel(model=model), True
+        return GrokModel(), True
     elif should_use_deepseek_model():
         return DeepSeekModel(model=model), True
+    elif should_use_anthropic_model():
+        return AnthropicModel(), True
     elif isinstance(model, str) or model is None:
         return GPTModel(model=model), True
 

deepeval/models/__init__.py

@@ -15,6 +15,7 @@ from deepeval.models.llms import (
     KimiModel,
     GrokModel,
     DeepSeekModel,
+    PortkeyModel,
 )
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
@@ -42,4 +43,5 @@ __all__ = [
     "AzureOpenAIEmbeddingModel",
     "LocalEmbeddingModel",
     "OllamaEmbeddingModel",
+    "PortkeyModel",
 ]

deepeval/models/llms/__init__.py

@@ -9,6 +9,7 @@ from .litellm_model import LiteLLMModel
 from .kimi_model import KimiModel
 from .grok_model import GrokModel
 from .deepseek_model import DeepSeekModel
+from .portkey_model import PortkeyModel
 
 __all__ = [
     "AzureOpenAIModel",
@@ -22,4 +23,5 @@ __all__ = [
     "KimiModel",
     "GrokModel",
     "DeepSeekModel",
+    "PortkeyModel",
 ]

deepeval/models/llms/amazon_bedrock_model.py

@@ -29,6 +29,7 @@ retry_bedrock = create_retry_decorator(PS.BEDROCK)
 
 _ALIAS_MAP = {
     "model": ["model_id"],
+    "region": ["region_name"],
     "cost_per_input_token": ["input_token_cost"],
     "cost_per_output_token": ["output_token_cost"],
 }
@@ -303,10 +304,10 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
         }
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        return (
-            input_tokens * self.cost_per_input_token
-            + output_tokens * self.cost_per_output_token
-        )
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     def load_model(self):
         pass

deepeval/models/llms/anthropic_model.py

@@ -227,9 +227,10 @@ class AnthropicModel(DeepEvalBaseLLM):
     ###############################################
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        input_cost = input_tokens * self.model_data.input_price
-        output_cost = output_tokens * self.model_data.output_price
-        return input_cost + output_cost
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     #########################
     # Capabilities          #

deepeval/models/llms/azure_model.py

@@ -386,9 +386,10 @@ class AzureOpenAIModel(DeepEvalBaseLLM):
     ###############################################
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        input_cost = input_tokens * self.model_data.input_price
-        output_cost = output_tokens * self.model_data.output_price
-        return input_cost + output_cost
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     ###############################################
     # Capabilities

deepeval/models/llms/deepseek_model.py

@@ -176,14 +176,11 @@ class DeepSeekModel(DeepEvalBaseLLM):
     # Utilities
     ###############################################
 
-    def calculate_cost(
-        self,
-        input_tokens: int,
-        output_tokens: int,
-    ) -> float:
-        input_cost = input_tokens * self.model_data.input_price
-        output_cost = output_tokens * self.model_data.output_price
-        return input_cost + output_cost
+    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     ###############################################
     # Capabilities

deepeval/models/llms/grok_model.py

@@ -224,14 +224,11 @@ class GrokModel(DeepEvalBaseLLM):
     # Utilities
     ###############################################
 
-    def calculate_cost(
-        self,
-        input_tokens: int,
-        output_tokens: int,
-    ) -> float:
-        input_cost = input_tokens * self.model_data.input_price
-        output_cost = output_tokens * self.model_data.output_price
-        return input_cost + output_cost
+    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     ###############################################
     # Capabilities

deepeval/models/llms/kimi_model.py

@@ -223,14 +223,11 @@ class KimiModel(DeepEvalBaseLLM):
     # Utilities
     ###############################################
 
-    def calculate_cost(
-        self,
-        input_tokens: int,
-        output_tokens: int,
-    ) -> float:
-        input_cost = input_tokens * self.model_data.input_price
-        output_cost = output_tokens * self.model_data.output_price
-        return input_cost + output_cost
+    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     ###############################################
     # Capabilities

deepeval/models/llms/litellm_model.py

@@ -289,6 +289,7 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 "top_logprobs": top_logprobs,
             }
             completion_params.update(self.kwargs)
+            completion_params.update(self.generation_kwargs)
 
             response = completion(**completion_params)
             cost = self.calculate_cost(response)
@@ -335,6 +336,7 @@ class LiteLLMModel(DeepEvalBaseLLM):
                 "top_logprobs": top_logprobs,
             }
             completion_params.update(self.kwargs)
+            completion_params.update(self.generation_kwargs)
 
             response = await acompletion(**completion_params)
             cost = self.calculate_cost(response)

deepeval/models/llms/local_model.py

@@ -52,7 +52,7 @@ class LocalModel(DeepEvalBaseLLM):
         self.base_url = (
             str(base_url).rstrip("/") if base_url is not None else None
         )
-        self.format = format or settings.LOCAL_MODEL_FORMAT
+        self.format = format or settings.LOCAL_MODEL_FORMAT or "json"
 
         if temperature is not None:
             temperature = float(temperature)

deepeval/models/llms/openai_model.py

@@ -378,9 +378,10 @@ class GPTModel(DeepEvalBaseLLM):
     #############
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        input_cost = input_tokens * self.model_data.input_price
-        output_cost = output_tokens * self.model_data.output_price
-        return input_cost + output_cost
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
 
     #########################
     # Capabilities          #

deepeval/models/retry_policy.py

@@ -87,6 +87,8 @@ def set_outer_deadline(seconds: float | None):
         call, which must be passed to `reset_outer_deadline` to restore the
         previous value.
     """
+    if get_settings().DEEPEVAL_DISABLE_TIMEOUTS:
+        return _OUTER_DEADLINE.set(None)
     if seconds and seconds > 0:
         return _OUTER_DEADLINE.set(time.monotonic() + seconds)
     return _OUTER_DEADLINE.set(None)
@@ -131,11 +133,10 @@ def resolve_effective_attempt_timeout():
         float: Seconds to use for the inner per-attempt timeout. `0` means
         disable inner timeout and rely on the outer budget instead.
     """
-    per_attempt = float(
-        get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
-    )
+    settings = get_settings()
+    per_attempt = float(settings.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
     # 0 or None disable inner wait_for. That means rely on outer task cap for timeouts instead.
-    if per_attempt <= 0:
+    if settings.DEEPEVAL_DISABLE_TIMEOUTS or per_attempt <= 0:
         return 0
     # If we do have a positive per-attempt, use up to remaining outer budget.
     rem = _remaining_budget()
@@ -557,7 +558,11 @@ def run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
         BaseException: If `func` raises, the same exception is re-raised with its
                        original traceback.
     """
-    if not timeout_seconds or timeout_seconds <= 0:
+    if (
+        get_settings().DEEPEVAL_DISABLE_TIMEOUTS
+        or not timeout_seconds
+        or timeout_seconds <= 0
+    ):
         return func(*args, **kwargs)
 
     # try to respect the global cap on concurrent timeout workers

deepeval/models/utils.py

@@ -123,11 +123,7 @@ def require_costs(
     # If model data doesn't have pricing, use provided values or environment variables
     if model_data.input_price is None or model_data.output_price is None:
         if cost_per_input_token is None or cost_per_output_token is None:
-            raise DeepEvalError(
-                f"No pricing available for `{model_name}`. "
-                f"Please provide both `cost_per_input_token` and `cost_per_output_token` when initializing `{model_name}`, "
-                f"or set {input_token_envvar} and {output_token_envvar} environment variables."
-            )
+            return None, None
 
         # Return the validated cost values as a tuple
         return cost_per_input_token, cost_per_output_token

deepeval/simulator/conversation_simulator.py

@@ -514,7 +514,9 @@ class ConversationSimulator:
     ):
         if not self.run_remote:
             conversation_history = json.dumps(
-                [t.model_dump() for t in turns], indent=4
+                [t.model_dump() for t in turns],
+                indent=4,
+                ensure_ascii=False,
             )
             prompt = self.template.stop_simulation(
                 conversation_history, golden.expected_outcome
@@ -559,7 +561,9 @@ class ConversationSimulator:
     ):
         if not self.run_remote:
             conversation_history = json.dumps(
-                [t.model_dump() for t in turns], indent=4
+                [t.model_dump() for t in turns],
+                indent=4,
+                ensure_ascii=False,
             )
             prompt = self.template.stop_simulation(
                 conversation_history, golden.expected_outcome

deepeval/simulator/template.py

@@ -57,7 +57,9 @@ class ConversationSimulatorTemplate:
         language: str,
     ) -> str:
         previous_conversation = json.dumps(
-            [t.model_dump() for t in turns], indent=4
+            [t.model_dump() for t in turns],
+            indent=4,
+            ensure_ascii=False,
         )
         prompt = textwrap.dedent(
             f"""