deepeval 3.7.9__py3-none-any.whl → 3.8.0__py3-none-any.whl

deepeval/models/llms/openrouter_model.py

@@ -0,0 +1,398 @@
+import warnings
+import inspect
+
+from typing import Optional, Tuple, Union, Dict, Type
+from pydantic import BaseModel, SecretStr
+from openai.types.chat.chat_completion import ChatCompletion
+from openai import (
+    OpenAI,
+    AsyncOpenAI,
+)
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.errors import DeepEvalError
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.models.llms.constants import DEFAULT_OPENROUTER_MODEL
+from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import require_secret_api_key
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
+
+
+retry_openrouter = create_retry_decorator(PS.OPENROUTER)
+
+
+def _request_timeout_seconds() -> float:
+    timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+    return timeout if timeout > 0 else 30.0
+
+
+def _convert_schema_to_openrouter_format(
+    schema: Union[Type[BaseModel], BaseModel],
+) -> Dict:
+    """
+    Convert Pydantic BaseModel to OpenRouter's JSON Schema format.
+
+    OpenRouter expects:
+    {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "schema_name",
+            "strict": true,
+            "schema": { ... JSON Schema ... }
+        }
+    }
+    """
+    json_schema = schema.model_json_schema()
+    schema_name = (
+        schema.__name__
+        if inspect.isclass(schema)
+        else schema.__class__.__name__
+    )
+
+    # OpenRouter requires additionalProperties: false when strict: true
+    # Ensure it's set at the root level of the schema
+    if "additionalProperties" not in json_schema:
+        json_schema["additionalProperties"] = False
+
+    return {
+        "type": "json_schema",
+        "json_schema": {
+            "name": schema_name,
+            "strict": True,
+            "schema": json_schema,
+        },
+    }
+
+
+class OpenRouterModel(DeepEvalBaseLLM):
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        temperature: Optional[float] = None,
+        cost_per_input_token: Optional[float] = None,
+        cost_per_output_token: Optional[float] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **kwargs,
+    ):
+        settings = get_settings()
+        model = model or settings.OPENROUTER_MODEL_NAME
+        if model is None:
+            model = DEFAULT_OPENROUTER_MODEL
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
+        else:
+            self.api_key = settings.OPENROUTER_API_KEY
+
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.OPENROUTER_BASE_URL is not None:
+            base_url = str(settings.OPENROUTER_BASE_URL).rstrip("/")
+        else:
+            base_url = "https://openrouter.ai/api/v1"
+
+        cost_per_input_token = (
+            cost_per_input_token
+            if cost_per_input_token is not None
+            else settings.OPENROUTER_COST_PER_INPUT_TOKEN
+        )
+        cost_per_output_token = (
+            cost_per_output_token
+            if cost_per_output_token is not None
+            else settings.OPENROUTER_COST_PER_OUTPUT_TOKEN
+        )
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        # validation
+        if temperature < 0:
+            raise DeepEvalError("Temperature must be >= 0.")
+
+        self.base_url = base_url
+        self.cost_per_input_token = cost_per_input_token
+        self.cost_per_output_token = cost_per_output_token
+        self.temperature = temperature
+
+        self.kwargs = dict(kwargs)
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
+        super().__init__(model)
+
+    ###############################################
+    # Generate functions
+    ###############################################
+
+    async def _generate_with_client(
+        self,
+        client: AsyncOpenAI,
+        prompt: str,
+        schema: Optional[BaseModel] = None,
+    ) -> Tuple[Union[str, Dict], float]:
+        """
+        Core generation logic shared between generate() and a_generate().
+
+        Args:
+            client: AsyncOpenAI client
+            prompt: The prompt to send
+            schema: Optional Pydantic schema for structured outputs
+
+        Returns:
+            Tuple of (output, cost)
+        """
+        if schema:
+            # Try OpenRouter's native JSON Schema format
+            try:
+                openrouter_response_format = (
+                    _convert_schema_to_openrouter_format(schema)
+                )
+                completion = await client.chat.completions.create(
+                    model=self.name,
+                    messages=[{"role": "user", "content": prompt}],
+                    response_format=openrouter_response_format,
+                    temperature=self.temperature,
+                    **self.generation_kwargs,
+                )
+
+                # Parse the JSON response and validate against schema
+                json_output = trim_and_load_json(
+                    completion.choices[0].message.content
+                )
+                cost = self.calculate_cost(
+                    completion.usage.prompt_tokens,
+                    completion.usage.completion_tokens,
+                    response=completion,
+                )
+                return schema.model_validate(json_output), cost
+            except Exception as e:
+                # Warn if structured outputs fail
+                warnings.warn(
+                    f"Structured outputs not supported for model '{self.name}'. "
+                    f"Falling back to regular generation with JSON parsing. "
+                    f"Error: {str(e)}",
+                    UserWarning,
+                    stacklevel=3,
+                )
+                # Fall back to regular generation and parse JSON manually (like Bedrock)
+                # This works with any model that can generate JSON in text
+                pass
+
+        # Regular generation (or fallback if structured outputs failed)
+        completion = await client.chat.completions.create(
+            model=self.name,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.temperature,
+            **self.generation_kwargs,
+        )
+
+        output = completion.choices[0].message.content
+        cost = self.calculate_cost(
+            completion.usage.prompt_tokens,
+            completion.usage.completion_tokens,
+            response=completion,
+        )
+        if schema:
+            # Parse JSON from text and validate against schema (like Bedrock)
+            json_output = trim_and_load_json(output)
+            return schema.model_validate(json_output), cost
+        else:
+            return output, cost
+
+    @retry_openrouter
+    def generate(
+        self, prompt: str, schema: Optional[BaseModel] = None
+    ) -> Tuple[Union[str, Dict], float]:
+        from deepeval.models.llms.utils import safe_asyncio_run
+
+        client = self.load_model(async_mode=True)
+        return safe_asyncio_run(
+            self._generate_with_client(client, prompt, schema)
+        )
+
+    @retry_openrouter
+    async def a_generate(
+        self, prompt: str, schema: Optional[BaseModel] = None
+    ) -> Tuple[Union[str, BaseModel], float]:
+        client = self.load_model(async_mode=True)
+        return await self._generate_with_client(client, prompt, schema)
+
+    ###############################################
+    # Other generate functions
+    ###############################################
+
+    @retry_openrouter
+    def generate_raw_response(
+        self,
+        prompt: str,
+        top_logprobs: int = 5,
+    ) -> Tuple[ChatCompletion, float]:
+        # Generate completion
+        client = self.load_model(async_mode=False)
+        completion = client.chat.completions.create(
+            model=self.name,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.temperature,
+            logprobs=True,
+            top_logprobs=top_logprobs,
+            **self.generation_kwargs,
+        )
+        # Cost calculation
+        input_tokens = completion.usage.prompt_tokens
+        output_tokens = completion.usage.completion_tokens
+        cost = self.calculate_cost(
+            input_tokens, output_tokens, response=completion
+        )
+
+        return completion, cost
+
+    @retry_openrouter
+    async def a_generate_raw_response(
+        self,
+        prompt: str,
+        top_logprobs: int = 5,
+    ) -> Tuple[ChatCompletion, float]:
+        # Generate completion
+        client = self.load_model(async_mode=True)
+        completion = await client.chat.completions.create(
+            model=self.name,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.temperature,
+            logprobs=True,
+            top_logprobs=top_logprobs,
+            **self.generation_kwargs,
+        )
+        # Cost calculation
+        input_tokens = completion.usage.prompt_tokens
+        output_tokens = completion.usage.completion_tokens
+        cost = self.calculate_cost(
+            input_tokens, output_tokens, response=completion
+        )
+
+        return completion, cost
+
+    @retry_openrouter
+    def generate_samples(
+        self, prompt: str, n: int, temperature: float
+    ) -> Tuple[list[str], float]:
+        client = self.load_model(async_mode=False)
+        response = client.chat.completions.create(
+            model=self.name,
+            messages=[{"role": "user", "content": prompt}],
+            n=n,
+            temperature=temperature,
+            **self.generation_kwargs,
+        )
+        completions = [choice.message.content for choice in response.choices]
+        cost = self.calculate_cost(
+            response.usage.prompt_tokens,
+            response.usage.completion_tokens,
+            response=response,
+        )
+        return completions, cost
+
+    ###############################################
+    # Utilities
+    ###############################################
+
+    def calculate_cost(
+        self, input_tokens: int, output_tokens: int, response=None
+    ) -> Optional[float]:
+        """
+        Calculate cost with priority:
+        1. User-provided pricing (highest priority)
+        2. Try to extract from API response (if OpenRouter includes pricing)
+        3. Return None if cost cannot be determined
+        """
+        # Priority 1: User-provided pricing
+        if (
+            self.cost_per_input_token is not None
+            and self.cost_per_output_token is not None
+        ):
+            return (
+                input_tokens * self.cost_per_input_token
+                + output_tokens * self.cost_per_output_token
+            )
+
+        # Priority 2: Try to extract from API response (if OpenRouter includes pricing)
+        # Note: OpenRouter may include pricing in response metadata
+        if response is not None:
+            # Check if response has cost information
+            usage_cost = getattr(getattr(response, "usage", None), "cost", None)
+            if usage_cost is not None:
+                try:
+                    return float(usage_cost)
+                except (ValueError, TypeError):
+                    pass
+            # Some responses might have cost at the top level
+            response_cost = getattr(response, "cost", None)
+            if response_cost is not None:
+                try:
+                    return float(response_cost)
+                except (ValueError, TypeError):
+                    pass
+
+        # Priority 3: Return None since cost is unknown
+        return None
+
+    ###############################################
+    # Model
+    ###############################################
+
+    def get_model_name(self):
+        return f"{self.name} (OpenRouter)"
+
+    def load_model(self, async_mode: bool = False):
+        if not async_mode:
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'openrouter' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.OPENROUTER):
+            kwargs["max_retries"] = 0
+
+        if not kwargs.get("timeout"):
+            kwargs["timeout"] = _request_timeout_seconds()
+
+        return kwargs
+
+    def _build_client(self, cls):
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="OpenRouter",
+            env_var_name="OPENROUTER_API_KEY",
+            param_hint="`api_key` to OpenRouterModel(...)",
+        )
+
+        kw = dict(
+            api_key=api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/retry_policy.py

@@ -772,6 +772,7 @@ AZURE_OPENAI_ERROR_POLICY = OPENAI_ERROR_POLICY
 DEEPSEEK_ERROR_POLICY = OPENAI_ERROR_POLICY
 KIMI_ERROR_POLICY = OPENAI_ERROR_POLICY
 LOCAL_ERROR_POLICY = OPENAI_ERROR_POLICY
+OPENROUTER_ERROR_POLICY = OPENAI_ERROR_POLICY
 
 ######################
 # AWS Bedrock Policy #
@@ -998,6 +999,7 @@ _POLICY_BY_SLUG: dict[str, Optional[ErrorPolicy]] = {
     PS.LITELLM.value: LITELLM_ERROR_POLICY,
     PS.LOCAL.value: LOCAL_ERROR_POLICY,
     PS.OLLAMA.value: OLLAMA_ERROR_POLICY,
+    PS.OPENROUTER.value: OPENROUTER_ERROR_POLICY,
 }
 
 
@@ -1019,6 +1021,7 @@ _STATIC_PRED_BY_SLUG: dict[str, Optional[Callable[[Exception], bool]]] = {
     PS.LITELLM.value: _opt_pred(LITELLM_ERROR_POLICY),
     PS.LOCAL.value: _opt_pred(LOCAL_ERROR_POLICY),
     PS.OLLAMA.value: _opt_pred(OLLAMA_ERROR_POLICY),
+    PS.OPENROUTER.value: _opt_pred(OPENROUTER_ERROR_POLICY),
 }
 
 

deepeval/prompt/api.py

@@ -30,6 +30,7 @@ class ModelProvider(Enum):
     X_AI = "X_AI"
     DEEPSEEK = "DEEPSEEK"
     BEDROCK = "BEDROCK"
+    OPENROUTER = "OPENROUTER"
 
 
 class ModelSettings(BaseModel):

deepeval/tracing/tracing.py

@@ -847,7 +847,12 @@ class Observer:
             self.trace_uuid = parent_span.trace_uuid
         else:
             current_trace = current_trace_context.get()
-            if current_trace:
+            # IMPORTANT: Verify trace is still active, not just in context
+            # (a previous failed async operation might leave a dead trace in context)
+            if (
+                current_trace
+                and current_trace.uuid in trace_manager.active_traces
+            ):
                 self.trace_uuid = current_trace.uuid
             else:
                 trace = trace_manager.start_new_trace(

deepeval/tracing/types.py

@@ -52,7 +52,7 @@ class LlmToolCall(BaseModel):
 
 class LlmOutput(BaseModel):
     role: str
-    content: str
+    content: Any
     tool_calls: Optional[List[LlmToolCall]] = None
 
 

{deepeval-3.7.9.dist-info → deepeval-3.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.7.9
+Version: 3.8.0
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -100,7 +100,7 @@ Description-Content-Type: text/markdown
     <a href="https://www.readme-i18n.com/confident-ai/deepeval?lang=zh">中文</a>
 </p>
 
-**DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating and testing large-language model systems. It is similar to Pytest but specialized for unit testing LLM outputs. DeepEval incorporates the latest research to evaluate LLM outputs based on metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which uses LLM-as-a-judge and other NLP models that runs **locally on your machine** for evaluation.
+**DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating and testing large-language model systems. It is similar to Pytest but specialized for unit testing LLM outputs. DeepEval incorporates the latest research to evaluate LLM outputs based on metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which uses LLM-as-a-judge and other NLP models that run **locally on your machine** for evaluation.
 
 Whether your LLM applications are AI agents, RAG pipelines, or chatbots, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily determine the optimal models, prompts, and architecture to improve your RAG pipeline, agentic workflows, prevent prompt drifting, or even transition from OpenAI to hosting your own Deepseek R1 with confidence.
 
@@ -118,7 +118,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
 > 🥳 You can now share DeepEval's test results on the cloud directly on [Confident AI](https://confident-ai.com?utm_source=GitHub)
 
 - Supports both end-to-end and component-level LLM evaluation.
-- Large variety of ready-to-use LLM evaluation metrics (all with explanations) powered by **ANY** LLM of your choice, statistical methods, or NLP models that runs **locally on your machine**:
+- Large variety of ready-to-use LLM evaluation metrics (all with explanations) powered by **ANY** LLM of your choice, statistical methods, or NLP models that run **locally on your machine**:
   - G-Eval
   - DAG ([deep acyclic graph](https://deepeval.com/docs/metrics-dag))
   - **RAG metrics:**

{deepeval-3.7.9.dist-info → deepeval-3.8.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deepeval/__init__.py,sha256=tle4lT4FONApg3OeztGPEdrpGMEGLWajyGTu7bEd3s0,2976
-deepeval/_version.py,sha256=IMF0EWAx17dGyvmjU00hlHiW4v1x9vnbFJs_IXEQ3gI,27
+deepeval/_version.py,sha256=zzwaJLAxT4xICJZz0E72TnWZ0UtmJr0uD8CK8vnXu6I,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -136,21 +136,21 @@ deepeval/benchmarks/winogrande/template.py,sha256=tDwH8NpNF9x7FbDmQw45XaW1LNqGBV
 deepeval/benchmarks/winogrande/winogrande.py,sha256=_4irJkRPw3c-Ufo-hM4cHpPKUoxozedFQpok9n0csTg,5644
 deepeval/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/cli/dotenv_handler.py,sha256=7PtVjCNUZKAXsVJQxznsLexad7y8x-gQ195xAxmv4gA,2468
-deepeval/cli/main.py,sha256=gni8-N6YX3zmEgkgr6eWE2_Vkb9fv2kLYJK3herJaTc,94957
+deepeval/cli/main.py,sha256=IsnAF8XVMA4j0dExMjlsD0ABPzFCwHkI4ai2S567qlA,100548
 deepeval/cli/server.py,sha256=cOm9xiYcPYB9GDeFQw9-Iawf9bNfOqftZs7q7mO_P7I,1979
 deepeval/cli/test.py,sha256=aoBPMfk0HTvOqb2xdvMykkx_s4SHst7lEnoUiSXo1lU,5483
 deepeval/cli/types.py,sha256=_7KdthstHNc-JKCWrfpDQCf_j8h9PMxh0qJCHmVXJr0,310
 deepeval/cli/utils.py,sha256=3fgH5WPTTe7Cz_QOLCHyflXB81kmFaSxXHJ2tnxvFLw,10649
 deepeval/confident/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-deepeval/confident/api.py,sha256=1eFM4umeBbyxNRNcKZqqp0g9rxklBfseHdbu7YDNx-g,8735
+deepeval/confident/api.py,sha256=3TpuZm59xo1_APsAPppreCRepf8pfGWksMmgxnwp764,8773
 deepeval/confident/types.py,sha256=9bgePDaU31yY7JGwCLZcc7pev9VGtNDZLbjsVpCLVdc,574
 deepeval/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/config/dotenv_handler.py,sha256=lOosoC7fm9RljriY8EFl5ywSGfSiQsVf_vmYqzpbZ8s,588
 deepeval/config/logging.py,sha256=ivqmhOSB-oHOOU3MvnhImrZwkkxzxKJgoKxesnWfHjg,1314
-deepeval/config/settings.py,sha256=uNFhulnSi07-uQfy6PxRPYK2EsCkdyq99fQCIOS_BpY,56583
+deepeval/config/settings.py,sha256=l5wvTEyG7KgWoY6F4Gc0j19oZef151r20Z8fZsiXR_U,57004
 deepeval/config/settings_manager.py,sha256=Ynebm2BKDrzajc6DEq2eYIwyRAAtUQOkTnl46albxLk,4187
 deepeval/config/utils.py,sha256=bJGljeAXoEYuUlYSvHSOsUnqINTwo6wOwfFHFpWxiaQ,4238
-deepeval/constants.py,sha256=J5rNXGsMKTFYJ_9Wi49qchZXuUityZjnvuy3I3TO5zk,1667
+deepeval/constants.py,sha256=MvwjLC1IHUY35FnnSsWVcHScmdbYBbPr8eTnsLWn40Y,1697
 deepeval/contextvars.py,sha256=oqXtuYiKd4Zvc1rNoR1gcRBxzZYCGTMVn7XostwvkRI,524
 deepeval/dataset/__init__.py,sha256=N2c-rkuxWYiiJSOZArw0H02Cwo7cnfzFuNYJlvsIBEg,249
 deepeval/dataset/api.py,sha256=bZ95HfIaxYB1IwTnp7x4AaKXWuII17T5uqVkhUXNc7I,1650
@@ -180,9 +180,9 @@ deepeval/integrations/hugging_face/rich_manager.py,sha256=WvFtPGpPmGeg2Ftsnojga6
 deepeval/integrations/hugging_face/tests/test_callbacks.py,sha256=88Wyg-aDaXujj9jHeGdFF3ITSl2-y7eaJGWgSyvvDi8,4607
 deepeval/integrations/hugging_face/utils.py,sha256=HUKdQcTIb76Ct69AS737oPxmlVxk5fw2UbT2pLn-o8k,1817
 deepeval/integrations/langchain/__init__.py,sha256=G1Qey5WkKou2-PA34KwWgmayQ_TbvXqPyotTbzmD8tw,84
-deepeval/integrations/langchain/callback.py,sha256=kN2AHdtvCL3XlmQTnwUQ8YEfklrpGciNGhqBpleoKQc,11938
+deepeval/integrations/langchain/callback.py,sha256=1K5KxpND6XEKCWnz-DWjhbO35AzmM3M8PIlk6bTYO2k,20360
 deepeval/integrations/langchain/patch.py,sha256=fCHfZXU9xX3IJ6SG8GEYzn3qrifyUkT0i_uUABTsmcs,1255
-deepeval/integrations/langchain/utils.py,sha256=6kuVsuepjPxj1sdlq1QgBeGhMByyXuNmb_FFflLVMhc,10070
+deepeval/integrations/langchain/utils.py,sha256=oYsQYO3Ucbmd1d7gkb5ARd60gm6BHtLy_5OOqUw8HIQ,11311
 deepeval/integrations/llama_index/__init__.py,sha256=Ujs9ZBJFkuCWUDBJOF88UbM1Y-S6QFQhxSo0oQnEWNw,90
 deepeval/integrations/llama_index/handler.py,sha256=uTvNXmAF4xBh8t9bBm5sBFX6ETp8SrkOZlFlE_GWdmM,10771
 deepeval/integrations/llama_index/utils.py,sha256=onmmo1vpn6cpOY5EhfTc0Uui7X6l1M0HD3sq-KVAesg,3380
@@ -191,7 +191,7 @@ deepeval/integrations/pydantic_ai/agent.py,sha256=-NKvpTUw3AxRNhuxVFcx9mw5BWCujz
 deepeval/integrations/pydantic_ai/instrumentator.py,sha256=Us9LSYZWMfaeAc7PGXMDYWzjWKFVmhRvZrFhSvmk448,11922
 deepeval/integrations/pydantic_ai/otel.py,sha256=CCqwCJ5pHqCzHgujHQqZy7Jxo2PH1BT0kR7QxdtzutY,2060
 deepeval/integrations/pydantic_ai/test_instrumentator.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/key_handler.py,sha256=x4lUC-zo18Xucf_sm3wJ5Xr-frSi4xscL5hO283JYr8,9247
+deepeval/key_handler.py,sha256=lajMBgF2lCzbQpW4e6Y7cD9FOw0Qk5UOKS4_kIIHj6Y,9562
 deepeval/metrics/__init__.py,sha256=19Df323r8aAlx2sRfV9BHJLicORhTLpogR8M1deJetw,4680
 deepeval/metrics/answer_relevancy/__init__.py,sha256=WbZUpoSg2GQoqJ4VIRirVVQ1JDx5xwT-RskwqNKfWGM,46
 deepeval/metrics/answer_relevancy/answer_relevancy.py,sha256=2zRAuPq794S-rqMesMnuvSJxd1yhEUTq8nRwLxf42QE,10958
@@ -233,7 +233,7 @@ deepeval/metrics/conversational_dag/conversational_dag.py,sha256=GlhJC-BBtnfx3G2
 deepeval/metrics/conversational_dag/nodes.py,sha256=xMhBEQ87CMWd6DPF-hMuMp3rxVuw_SS7mHKHqGSfxLw,31348
 deepeval/metrics/conversational_dag/templates.py,sha256=zMF9rjRQCbAtMXXC03m1CwcYiYwsRCfoitf63QkvCmE,4243
 deepeval/metrics/conversational_g_eval/__init__.py,sha256=0whQUYv_qZx4nkz0V6TTjgg8gJVCmW8323WcXpj2EzI,93
-deepeval/metrics/conversational_g_eval/conversational_g_eval.py,sha256=eHII40aZQTfA-zJUKYOMeOsli9ak_CeTCTFe27iBHbY,16143
+deepeval/metrics/conversational_g_eval/conversational_g_eval.py,sha256=R6qt6yO8_k0ciXUz3F_imMxS9hFi0EtMgDkooOyE790,17190
 deepeval/metrics/conversational_g_eval/schema.py,sha256=H_9-iA1BXJwbPKrGEZBqxDO_En4sjXI8_xKSNYc-hnk,167
 deepeval/metrics/conversational_g_eval/template.py,sha256=JVKwZJBgHiP1cMuGTLAL_taKvRL-ppJjkiTOs0wzgYk,2931
 deepeval/metrics/dag/__init__.py,sha256=G5D9ngJ6nnbRBF2mfmNZymZId8gKD09QzTA1Y_bTrgM,157
@@ -250,10 +250,10 @@ deepeval/metrics/faithfulness/faithfulness.py,sha256=2mFXIh0U7Xf4Ybl6w-Lt74D4P13
 deepeval/metrics/faithfulness/schema.py,sha256=yPbe1CrW6PMOJjnWnUOxeb_Ul8sfDwvwgt4QTW-95RI,437
 deepeval/metrics/faithfulness/template.py,sha256=n9SvOM8iJ9Y4K0o8OHHc0uyw3E_v2BfZRd0fSUIX8XI,11126
 deepeval/metrics/g_eval/__init__.py,sha256=HAhsQFVq9LIpZXPN00Jc_WrMXrh47NIT86VnUpWM4_4,102
-deepeval/metrics/g_eval/g_eval.py,sha256=kG-8V46pkdope2ptQT587CT7_Z-AlwKkzV_16Y2Jbrs,15117
+deepeval/metrics/g_eval/g_eval.py,sha256=VlQkYuWwWITB0wo8q1OVZEQjZ7V7gpDv4kdvrUP3ROA,16134
 deepeval/metrics/g_eval/schema.py,sha256=V629txuDrr_2IEKEsgJVYYZb_pkdfcltQV9ZjvxK5co,287
 deepeval/metrics/g_eval/template.py,sha256=v96BJFOH1rnME6b-OwJwcunvA4dd2GwraoXnjiZRu9Y,5182
-deepeval/metrics/g_eval/utils.py,sha256=GultslyqO70msvORp6yjPcvlVKMA8vL4nQNnARdH9bM,8809
+deepeval/metrics/g_eval/utils.py,sha256=t8SvFt_2GRSOOKiYMHJWyhrzLCWOdFWGnlQ38PZA0Ls,10767
 deepeval/metrics/goal_accuracy/__init__.py,sha256=SVvA5Py1iNQoLujNUptvckoLoR6XMs-W2jQ7b89v-Tc,46
 deepeval/metrics/goal_accuracy/goal_accuracy.py,sha256=zlzxrAQQ4ASrjVo4-jDmn6uBw50sHOsAtgWez0CpveU,13034
 deepeval/metrics/goal_accuracy/schema.py,sha256=WmP1nw5ugOAKCFrpjZpF4jjeJzLB3Ecdp-2VWfBJLAE,257
@@ -388,7 +388,7 @@ deepeval/metrics/utils.py,sha256=RS8gsEh__DaKhXjdDfNcw1iOVvN40Z1mbQHM21Q30Iw,213
 deepeval/model_integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/model_integrations/types.py,sha256=rbVMhC_2yWwD6JqzkRO9D7aMVC_KtXN686G_S7de7S8,630
 deepeval/model_integrations/utils.py,sha256=Zt9SYPgTxlGsQFZgpZvh_a5fWuL8mmIFVSe6uoQywZ4,3562
-deepeval/models/__init__.py,sha256=NLG4t8GevFwGac3xdnGVZbPx-ysbeghHJZ4rdv4DQ4c,980
+deepeval/models/__init__.py,sha256=-eHIhBn1a1fLKjPd5TCW8da8UuCKYdJvwic26swFtZM,1024
 deepeval/models/_summac_model.py,sha256=xflanxl_IBuzuainlYCVX7UvjHCnAckKSvNR2NwZI6k,19750
 deepeval/models/answer_relevancy_model.py,sha256=SLOA6uUImNOuxpPGfTg2AH7MIkf9QsotYixvI1jcVC8,2197
 deepeval/models/base_model.py,sha256=uja2bZcrTCIPMkIDgOLG2k2Ncw7uTX6vjvnrQtFlNlk,4891
@@ -399,11 +399,11 @@ deepeval/models/embedding_models/local_embedding_model.py,sha256=Io5dYNR8f-_iMmz
 deepeval/models/embedding_models/ollama_embedding_model.py,sha256=4uxrzdBlpWT-SM1HHHsZXhwg1ejkxUDFewxACLeYsG4,3747
 deepeval/models/embedding_models/openai_embedding_model.py,sha256=S8uvWODbiTF4EYfeID5yEF0YvYkDs1dP_Kiur4sb67M,4477
 deepeval/models/hallucination_model.py,sha256=ABi978VKLE_jNHbDzM96kJ08EsZ5ZlvOlJHA_ptSkfQ,1003
-deepeval/models/llms/__init__.py,sha256=vrz8j4AjRDOHwkUPokXgukls_hMJkNgmphCVqDMns28,730
+deepeval/models/llms/__init__.py,sha256=Mlkvw9eIbxJXJjTB9Nj0LoL-kSRCmewrEihDvFyzvJA,799
 deepeval/models/llms/amazon_bedrock_model.py,sha256=mgBdGhyZo0SU0OMzqWAF6pUlQMUuCCdDiordfinDNpM,10898
 deepeval/models/llms/anthropic_model.py,sha256=08_nGK5EoGpf_F0I6JkhrEAswDc9DjLQqGYMX3emsoQ,10542
 deepeval/models/llms/azure_model.py,sha256=Nc_LgA8rEhkldvdhccNojERaviaBg6jyfBVL9bGdKek,16673
-deepeval/models/llms/constants.py,sha256=0qnrd1bC46BdTAaf4Ja9WkTFYNysf6idsn-6s0Z5BtA,71143
+deepeval/models/llms/constants.py,sha256=H6_FyTNkfF0wr3R8qUlvT2LuZGT5lbXFh9Hcq5T8A8k,72008
 deepeval/models/llms/deepseek_model.py,sha256=OzEs0hnSixqICurVFo6T5GBAUeDrnWOlooEyJrgi5zE,8565
 deepeval/models/llms/gemini_model.py,sha256=h01bJnLBnc1xaqoVSBOL-PwllCsHRkA88bp21BA0Mws,15552
 deepeval/models/llms/grok_model.py,sha256=zGU1WzKADrgap5NQJTDb6BY4SZNNJqAZ6phnK_HFJqw,10703
@@ -411,10 +411,11 @@ deepeval/models/llms/kimi_model.py,sha256=n5w2MeeKSMS7HvSpiDSQueZ2EQSv3c6pDb-C-A
 deepeval/models/llms/litellm_model.py,sha256=lWfJvzWia7XCrLiRTNF0fUQXYOalsLV1y3Tq03loDP4,16533
 deepeval/models/llms/local_model.py,sha256=1KWuvgfdSE2XaNriIz-8gBIrbvmLOgomZxXI80Zt-8c,8287
 deepeval/models/llms/ollama_model.py,sha256=fk2GlQSFMYBe9oKrFouWAIf_PtSZZp8SGV2HXpE66no,7957
-deepeval/models/llms/openai_model.py,sha256=uXk3jsBjF1AscWkT3O-FfSosgn-L91wIQU9wvCBqokc,17227
+deepeval/models/llms/openai_model.py,sha256=3cp7fkpnh6suMu9lWTe0rYASALqWIZa2aUHVQhL8JHM,17243
+deepeval/models/llms/openrouter_model.py,sha256=hXDPQGa-HrvkXBL1E8ju1pduqsAnZg86RbotC0NcENU,13617
 deepeval/models/llms/portkey_model.py,sha256=EvUJDHhtKzetFakc1HzxYIpzUlgD3UnEvZr0q9hpIKU,6684
 deepeval/models/llms/utils.py,sha256=NsrZ4DjrVBnYbMZEY6G8U9On_B84kiDkB88yeyBUFfw,1433
-deepeval/models/retry_policy.py,sha256=lFon21lgFmgMTryd6BRtaCTrcIGUVobP1MOjWtKPgnI,34816
+deepeval/models/retry_policy.py,sha256=hNQRpP6SL5rpFTvLL-BRXkBLknmTwusJXuZnW9ba_as,34973
 deepeval/models/summac_model.py,sha256=wKeH7pWQRXrTlzlIw_r1YCb8b7jUhWq6jUz9FiNUCSg,1992
 deepeval/models/unbias_model.py,sha256=umOMhQLTmnD7uOuhiQufEl4Wlti4q2s3EtKOpds7zhs,597
 deepeval/models/utils.py,sha256=0_6_hmEwatWGoba-KNE38KvmDKMlhGff2lIzOCpRQgQ,5947
@@ -459,7 +460,7 @@ deepeval/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 deepeval/plugins/plugin.py,sha256=_dwsdx4Dg9DbXxK3f7zJY4QWTJQWc7QE1HmIg2Zjjag,1515
 deepeval/progress_context.py,sha256=ZSKpxrE9sdgt9G3REKnVeXAv7GJXHHVGgLynpG1Pudw,3557
 deepeval/prompt/__init__.py,sha256=rDU99KjydxDRKhuQJCBs_bpDJrWb2mpHtvyv6AEwFC8,367
-deepeval/prompt/api.py,sha256=JLQFJFKsJSadCnp7vLvapmPoCoVyBbEsXZrxPAolpiA,6329
+deepeval/prompt/api.py,sha256=DNhKouq3ntEKmN_VegNh5X1gu_2RGJwzBp07rEEyg6s,6359
 deepeval/prompt/prompt.py,sha256=waaQDrTXQQUzOIJbOYtUpoa4qsuXgmzObUwFH-wRx2Y,31654
 deepeval/prompt/utils.py,sha256=knjgPU2066OtYWMb3NqMPChr9zQgKfXo_QTLTtSkmYg,7620
 deepeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -515,12 +516,12 @@ deepeval/tracing/patchers.py,sha256=Oi9wao3oDYhcviv7p0KoWBeS9ne7rHLa2gh9AR9EyiU,
 deepeval/tracing/perf_epoch_bridge.py,sha256=iyAPddB6Op7NpMtPHJ29lDm53Btz9yLaN6xSCfTRQm4,1825
 deepeval/tracing/trace_context.py,sha256=Z0n0Cu1A5g9dXiZnzTFO5TzeOYHKeNuO6v3_EU_Gi_c,3568
 deepeval/tracing/trace_test_manager.py,sha256=wt4y7EWTRc4Bw938-UFFtXHkdFFOrnx6JaIk7J5Iulw,555
-deepeval/tracing/tracing.py,sha256=VCvFrPdBPSu1tL6xGswgOcCGBk-bU25wrcWAKD_8Uf0,46465
-deepeval/tracing/types.py,sha256=WhnxefUc5I8jcAOBQ-tsZ8_zVZfGqSvCWHD5XUN6Ggw,6040
+deepeval/tracing/tracing.py,sha256=AkbmgjWzSQ2k2qeN9i8LT17MsafuBenzzkP0r31I950,46728
+deepeval/tracing/types.py,sha256=3QkF0toQ6f0fEDARYOUV6Iv9UJFbg14kSpn3dL1H5CE,6040
 deepeval/tracing/utils.py,sha256=mdvhYAxDNsdnusaEXJd-c-_O2Jn6S3xSuzRvLO1Jz4U,5684
 deepeval/utils.py,sha256=Wsu95g6t1wdttxWIESVwuUxbml7C-9ZTsV7qHCQI3Xg,27259
-deepeval-3.7.9.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.7.9.dist-info/METADATA,sha256=KnhwryGSEXENiHndjsgpc2f9v0ALbkufYyD-6kzKg6s,18754
-deepeval-3.7.9.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.7.9.dist-info/entry_points.txt,sha256=NoismUQfwLOojSGZmBrdcpwfaoFRAzUhBvZD3UwOKog,95
-deepeval-3.7.9.dist-info/RECORD,,
+deepeval-3.8.0.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.8.0.dist-info/METADATA,sha256=z7OpguZITPdv0S9jhvE6CEq-zVCSq9fvbKChLKD9gwc,18752
+deepeval-3.8.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.8.0.dist-info/entry_points.txt,sha256=NoismUQfwLOojSGZmBrdcpwfaoFRAzUhBvZD3UwOKog,95
+deepeval-3.8.0.dist-info/RECORD,,