deepeval 3.4.8__py3-none-any.whl → 3.4.9__py3-none-any.whl

deepeval/models/llms/amazon_bedrock_model.py

@@ -115,13 +115,34 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
     ###############################################
 
     def get_converse_request_body(self, prompt: str) -> dict:
+        # Inline parameter translation with defaults
+        param_mapping = {
+            "max_tokens": "maxTokens",
+            "top_p": "topP",
+            "top_k": "topK",
+            "stop_sequences": "stopSequences",
+        }
+
+        # Start with defaults for required parameters
+        translated_kwargs = {
+            "maxTokens": self.generation_kwargs.get("max_tokens", 1000),
+            "topP": self.generation_kwargs.get("top_p", 0),
+        }
+
+        # Add any other parameters from generation_kwargs
+        for key, value in self.generation_kwargs.items():
+            if key not in [
+                "max_tokens",
+                "top_p",
+            ]:  # Skip already handled defaults
+                aws_key = param_mapping.get(key, key)
+                translated_kwargs[aws_key] = value
+
         return {
             "messages": [{"role": "user", "content": [{"text": prompt}]}],
             "inferenceConfig": {
                 "temperature": self.temperature,
-                "topP": self.generation_kwargs.get("top_p", 0),
-                "maxTokens": self.generation_kwargs.get("max_tokens", 1000),
-                **self.generation_kwargs,
+                **translated_kwargs,
             },
         }
 

deepeval/models/llms/openai_model.py

@@ -1,26 +1,33 @@
+import logging
+
 from openai.types.chat.chat_completion import ChatCompletion
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from typing import Optional, Tuple, Union, Dict
-from openai import OpenAI, AsyncOpenAI
 from pydantic import BaseModel
-import logging
-import openai
 
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    wait_exponential_jitter,
-    RetryCallState,
+from openai import (
+    OpenAI,
+    AsyncOpenAI,
 )
 
+from tenacity import retry, RetryCallState, before_sleep_log
+
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models.utils import parse_model_name
+from deepeval.models.retry_policy import (
+    OPENAI_ERROR_POLICY,
+    default_wait,
+    default_stop,
+    retry_predicate,
+)
+
+logger = logging.getLogger("deepeval.openai_model")
 
 
 def log_retry_error(retry_state: RetryCallState):
     exception = retry_state.outcome.exception()
-    logging.error(
+    logger.error(
         f"OpenAI Error: {exception} Retrying: {retry_state.attempt_number} time(s)..."
     )
 
@@ -212,14 +219,22 @@ models_requiring_temperature_1 = [
     "gpt-5-chat-latest",
 ]
 
-retryable_exceptions = (
-    openai.RateLimitError,
-    openai.APIConnectionError,
-    openai.APITimeoutError,
-    openai.LengthFinishReasonError,
+_base_retry_rules_kw = dict(
+    wait=default_wait(),
+    stop=default_stop(),
+    retry=retry_predicate(OPENAI_ERROR_POLICY),
+    before_sleep=before_sleep_log(
+        logger, logging.INFO
+    ),  # <- logs only on retries
+    after=log_retry_error,
 )
 
 
+def _openai_client_kwargs():
+    # Avoid double-retry at SDK layer by disabling the SDK's own retries so tenacity is the single source of truth for retry logic.
+    return {"max_retries": 0}
+
+
 class GPTModel(DeepEvalBaseLLM):
     def __init__(
         self,
@@ -296,11 +311,7 @@ class GPTModel(DeepEvalBaseLLM):
     # Generate functions
     ###############################################
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry(**_base_retry_rules_kw)
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -359,11 +370,7 @@ class GPTModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry(**_base_retry_rules_kw)
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, BaseModel], float]:
@@ -427,11 +434,7 @@ class GPTModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry(**_base_retry_rules_kw)
     def generate_raw_response(
         self,
         prompt: str,
@@ -454,11 +457,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         return completion, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry(**_base_retry_rules_kw)
     async def a_generate_raw_response(
         self,
         prompt: str,
@@ -481,11 +480,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         return completion, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry(**_base_retry_rules_kw)
     def generate_samples(
         self, prompt: str, n: int, temperature: float
     ) -> Tuple[list[str], float]:
@@ -518,12 +513,13 @@ class GPTModel(DeepEvalBaseLLM):
         return self.model_name
 
     def load_model(self, async_mode: bool = False):
+        kwargs = {**self.kwargs, **_openai_client_kwargs()}
         if not async_mode:
             return OpenAI(
                 api_key=self._openai_api_key,
                 base_url=self.base_url,
-                **self.kwargs,
+                **kwargs,
             )
         return AsyncOpenAI(
-            api_key=self._openai_api_key, base_url=self.base_url, **self.kwargs
+            api_key=self._openai_api_key, base_url=self.base_url, **kwargs
         )

deepeval/models/retry_policy.py

@@ -0,0 +1,280 @@
+"""Generic retry policy helpers for provider SDKs.
+
+This module lets models define *what is transient* vs *non-retryable* (permanent) failure
+without coupling to a specific SDK. You provide an `ErrorPolicy` describing
+exception classes and special “non-retryable” error codes, such as quota-exhausted from OpenAI,
+and get back a Tenacity predicate suitable for `retry_if_exception`.
+
+Typical use:
+
+    # Import dependencies
+    from tenacity import retry, before_sleep_log
+    from deepeval.models.retry_policy import (
+        OPENAI_ERROR_POLICY, default_wait, default_stop, retry_predicate
+    )
+
+    # Define retry rule keywords
+    _retry_kw = dict(
+        wait=default_wait(),
+        stop=default_stop(),
+        retry=retry_predicate(OPENAI_ERROR_POLICY),
+        before_sleep=before_sleep_log(logger, logging.INFO), # <- Optional: logs only on retries
+    )
+
+    # Apply retry rule keywords where desired
+    @retry(**_retry_kw)
+    def call_openai(...):
+        ...
+"""
+
+from __future__ import annotations
+
+import logging
+
+from deepeval.utils import read_env_int, read_env_float
+from dataclasses import dataclass, field
+from typing import Iterable, Mapping, Callable, Sequence, Tuple
+from collections.abc import Mapping as ABCMapping
+from tenacity import (
+    wait_exponential_jitter,
+    stop_after_attempt,
+    retry_if_exception,
+)
+
+
+logger = logging.getLogger(__name__)
+
+# --------------------------
+# Policy description
+# --------------------------
+
+
+@dataclass(frozen=True)
+class ErrorPolicy:
+    """Describe exception classes & rules for retry classification.
+
+    Attributes:
+        auth_excs: Exceptions that indicate authentication/authorization problems.
+                   These are treated as non-retryable.
+        rate_limit_excs: Exceptions representing rate limiting (HTTP 429).
+        network_excs: Exceptions for timeouts / connection issues (transient).
+        http_excs: Exceptions carrying an integer `status_code` (4xx, 5xx)
+        non_retryable_codes: Error “code” strings that should be considered permanent,
+                             such as "insufficient_quota". Used to refine rate-limit handling.
+        retry_5xx: Whether to retry provider 5xx responses (defaults to True).
+    """
+
+    auth_excs: Tuple[type[Exception], ...]
+    rate_limit_excs: Tuple[type[Exception], ...]
+    network_excs: Tuple[type[Exception], ...]
+    http_excs: Tuple[type[Exception], ...]
+    non_retryable_codes: frozenset[str] = field(default_factory=frozenset)
+    retry_5xx: bool = True
+    message_markers: Mapping[str, Iterable[str]] = field(default_factory=dict)
+
+
+# --------------------------
+# Extraction helpers
+# --------------------------
+
+
+def extract_error_code(
+    e: Exception,
+    *,
+    response_attr: str = "response",
+    body_attr: str = "body",
+    code_path: Sequence[str] = ("error", "code"),
+    message_markers: Mapping[str, Iterable[str]] | None = None,
+) -> str:
+    """Best effort extraction of an error 'code' for SDK compatibility.
+
+    Order of attempts:
+      1) Structured JSON via `e.response.json()` (typical HTTP error payload).
+      2) A dict stored on `e.body` (some gateways/proxies use this).
+      3) Message sniffing fallback, using `message_markers`.
+
+    Args:
+        e: The exception raised by the SDK/provider client.
+        response_attr: Attribute name that holds an HTTP response object.
+        body_attr: Attribute name that may hold a parsed payload (dict).
+        code_path: Path of keys to traverse to the code (e.g., ["error", "code"]).
+        message_markers: Mapping from canonical code -> substrings to search for.
+
+    Returns:
+        The code string if found, else "".
+    """
+    # 1) Structured JSON in e.response.json()
+    resp = getattr(e, response_attr, None)
+    if resp is not None:
+        try:
+            cur = resp.json()
+            for k in code_path:
+                if not isinstance(cur, ABCMapping):
+                    cur = {}
+                    break
+                cur = cur.get(k, {})
+            if isinstance(cur, (str, int)):
+                return str(cur)
+        except Exception:
+            # response.json() can raise; ignore and fall through
+            pass
+
+    # 2) SDK provided dict body
+    body = getattr(e, body_attr, None)
+    if isinstance(body, ABCMapping):
+        cur = body
+        for k in code_path:
+            if not isinstance(cur, ABCMapping):
+                cur = {}
+                break
+            cur = cur.get(k, {})
+        if isinstance(cur, (str, int)):
+            return str(cur)
+
+    # 3) Message sniff (hopefully this helps catch message codes that slip past the previous 2 parsers)
+    msg = str(e).lower()
+    markers = message_markers or {}
+    for code_key, needles in markers.items():
+        if any(n in msg for n in needles):
+            return code_key
+
+    return ""
+
+
+# --------------------------
+# Predicate factory
+# --------------------------
+
+
+def make_is_transient(
+    policy: ErrorPolicy,
+    *,
+    message_markers: Mapping[str, Iterable[str]] | None = None,
+    extra_non_retryable_codes: Iterable[str] = (),
+) -> Callable[[Exception], bool]:
+    """Create a Tenacity predicate: True = retry, False = surface immediately.
+
+    Semantics:
+        - Auth errors: non-retryable.
+        - Rate limit errors: retry unless the extracted code is in the non-retryable set
+        - Network/timeout errors: retry.
+        - HTTP errors with a `status_code`: retry 5xx if `policy.retry_5xx` is True.
+        - Everything else: treated as non-retryable.
+
+    Args:
+        policy: An ErrorPolicy describing error classes and rules.
+        message_markers: Optional override/extension for code inference via message text.
+        extra_non_retryable_codes: Additional code strings to treat as non-retryable.
+
+    Returns:
+        A callable `predicate(e) -> bool` suitable for `retry_if_exception`.
+    """
+    non_retryable = frozenset(policy.non_retryable_codes) | frozenset(
+        extra_non_retryable_codes
+    )
+
+    def _pred(e: Exception) -> bool:
+        if isinstance(e, policy.auth_excs):
+            return False
+
+        if isinstance(e, policy.rate_limit_excs):
+            code = extract_error_code(
+                e, message_markers=(message_markers or policy.message_markers)
+            )
+            return code not in non_retryable
+
+        if isinstance(e, policy.network_excs):
+            return True
+
+        if isinstance(e, policy.http_excs):
+            try:
+                sc = int(getattr(e, "status_code", 0))
+            except Exception:
+                sc = 0
+            return policy.retry_5xx and 500 <= sc < 600
+
+        return False
+
+    return _pred
+
+
+# --------------------------
+# Tenacity convenience
+# --------------------------
+
+
+def default_wait():
+    """Default backoff: exponential with jitter, capped.
+    Overridable via env:
+      - DEEPEVAL_RETRY_INITIAL_SECONDS (>=0)
+      - DEEPEVAL_RETRY_EXP_BASE      (>=1)
+      - DEEPEVAL_RETRY_JITTER        (>=0)
+      - DEEPEVAL_RETRY_CAP_SECONDS   (>=0)
+    """
+    initial = read_env_float(
+        "DEEPEVAL_RETRY_INITIAL_SECONDS", 1.0, min_value=0.0
+    )
+    exp_base = read_env_float("DEEPEVAL_RETRY_EXP_BASE", 2.0, min_value=1.0)
+    jitter = read_env_float("DEEPEVAL_RETRY_JITTER", 2.0, min_value=0.0)
+    cap = read_env_float("DEEPEVAL_RETRY_CAP_SECONDS", 5.0, min_value=0.0)
+    return wait_exponential_jitter(
+        initial=initial, exp_base=exp_base, jitter=jitter, max=cap
+    )
+
+
+def default_stop():
+    """Default stop condition: at most N attempts (N-1 retries).
+    Overridable via env:
+      - DEEPEVAL_RETRY_MAX_ATTEMPTS (>=1)
+    """
+    attempts = read_env_int("DEEPEVAL_RETRY_MAX_ATTEMPTS", 2, min_value=1)
+    return stop_after_attempt(attempts)
+
+
+def retry_predicate(policy: ErrorPolicy, **kw):
+    """Build a Tenacity `retry=` argument from a policy.
+
+    Example:
+        retry=retry_predicate(OPENAI_ERROR_POLICY, extra_non_retryable_codes=["some_code"])
+    """
+    return retry_if_exception(make_is_transient(policy, **kw))
+
+
+# --------------------------
+# Built-in policies
+# --------------------------
+OPENAI_MESSAGE_MARKERS: dict[str, tuple[str, ...]] = {
+    "insufficient_quota": ("insufficient_quota", "exceeded your current quota"),
+}
+
+try:
+    from openai import (
+        AuthenticationError,
+        RateLimitError,
+        APIConnectionError,
+        APITimeoutError,
+        APIStatusError,
+    )
+
+    OPENAI_ERROR_POLICY = ErrorPolicy(
+        auth_excs=(AuthenticationError,),
+        rate_limit_excs=(RateLimitError,),
+        network_excs=(APIConnectionError, APITimeoutError),
+        http_excs=(APIStatusError,),
+        non_retryable_codes=frozenset({"insufficient_quota"}),
+        message_markers=OPENAI_MESSAGE_MARKERS,
+    )
+except Exception:  # pragma: no cover - OpenAI may not be installed in some envs
+    OPENAI_ERROR_POLICY = None
+
+
+__all__ = [
+    "ErrorPolicy",
+    "extract_error_code",
+    "make_is_transient",
+    "default_wait",
+    "default_stop",
+    "retry_predicate",
+    "OPENAI_MESSAGE_MARKERS",
+    "OPENAI_ERROR_POLICY",
+]

deepeval/openai_agents/agent.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field, replace
-from typing import Any, Optional, Awaitable, Callable
+from typing import Any, Optional, Awaitable, Callable, Generic, TypeVar
 
 from deepeval.tracing import observe
 from deepeval.prompt import Prompt
@@ -14,6 +14,8 @@ except Exception as e:
         "openai-agents is required for this integration. Please install it."
     ) from e
 
+TContext = TypeVar("TContext")
+
 
 class _ObservedModel(Model):
     def __init__(
@@ -153,7 +155,7 @@ class _ObservedProvider(ModelProvider):
 
 
 @dataclass
-class DeepEvalAgent(BaseAgent[Any]):
+class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
     """
     A subclass of agents.Agent that accepts `metrics` and `metric_collection`
     and ensures the underlying model's `get_response` is wrapped with deepeval.observe.

deepeval/test_run/api.py

@@ -50,6 +50,7 @@ class LLMApiTestCase(BaseModel):
     trace: Optional[TraceApi] = Field(None)
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
+    # metric_collection: Optional[str] = Field(None, alias="metricCollection")
 
     def update_metric_data(self, metric_data: MetricData):
         if self.metrics_data is None:

deepeval/tracing/otel/exporter.py

@@ -30,6 +30,7 @@ from deepeval.tracing.otel.utils import (
     to_hex_string,
     parse_string,
     parse_list_of_strings,
+    post_test_run,
 )
 from deepeval.tracing import perf_epoch_bridge as peb
 from deepeval.tracing.types import TraceAttributes
@@ -80,7 +81,8 @@ class ConfidentSpanExporter(SpanExporter):
         self,
         spans: typing.Sequence[ReadableSpan],
         timeout_millis: int = 30000,
-        api_key: Optional[str] = None,  # dynamic api key
+        api_key: Optional[str] = None,  # dynamic api key,
+        _test_run_id: Optional[str] = None,
     ) -> SpanExportResult:
         # build forest of spans
         forest = self._build_span_forest(spans)
@@ -223,14 +225,24 @@ class ConfidentSpanExporter(SpanExporter):
                 trace_manager.add_span_to_trace(base_span_wrapper.base_span)
                 # no removing span because it can be parent of other spans
 
-        # safely end all active traces
+        # safely end all active traces or return them for test runs
         active_traces_keys = list(trace_manager.active_traces.keys())
-        for trace_key in active_traces_keys:
-            set_trace_time(trace_manager.get_trace_by_uuid(trace_key))
-            trace_manager.end_trace(trace_key)
-        trace_manager.clear_traces()
-
-        return SpanExportResult.SUCCESS
+        if _test_run_id:
+            traces = []
+            for trace_key in active_traces_keys:
+                set_trace_time(trace_manager.get_trace_by_uuid(trace_key))
+                trace = trace_manager.get_trace_by_uuid(trace_key)
+                if trace:
+                    traces.append(trace)
+            trace_manager.clear_traces()
+            post_test_run(traces, _test_run_id)
+            return SpanExportResult.SUCCESS
+        else:
+            for trace_key in active_traces_keys:
+                set_trace_time(trace_manager.get_trace_by_uuid(trace_key))
+                trace_manager.end_trace(trace_key)
+            trace_manager.clear_traces()
+            return SpanExportResult.SUCCESS
 
     def _convert_readable_span_to_base_span(
         self, span: ReadableSpan

deepeval/tracing/otel/utils.py

@@ -1,5 +1,6 @@
 from typing import List, Optional, Tuple, Any
 from deepeval.tracing.types import Trace, LLMTestCase, ToolCall
+from deepeval.tracing import trace_manager, BaseSpan
 from opentelemetry.sdk.trace.export import ReadableSpan
 import json
 
@@ -250,3 +251,59 @@ def parse_list_of_strings(context: List[str]) -> List[str]:
             else:
                 parsed_context.append(context_str)
     return parsed_context
+
+
+from deepeval.evaluate.utils import create_api_test_case
+from deepeval.test_run.api import LLMApiTestCase
+from deepeval.test_run.test_run import global_test_run_manager
+from typing import Optional
+
+
+def post_test_run(traces: List[Trace], test_run_id: Optional[str]):
+    # Accept single trace or list of traces
+    if isinstance(traces, Trace):
+        traces = [traces]
+
+    api_test_cases: List[LLMApiTestCase] = []
+
+    # Collect test cases from spans that have metric_collection
+    for trace in traces:
+        trace_api = trace_manager.create_trace_api(trace)
+
+        def dfs(span: BaseSpan):
+            if span.metric_collection:
+                llm_test_case = LLMTestCase(
+                    input=str(span.input),
+                    actual_output=(
+                        str(span.output) if span.output is not None else None
+                    ),
+                    expected_output=span.expected_output,
+                    context=span.context,
+                    retrieval_context=span.retrieval_context,
+                    tools_called=span.tools_called,
+                    expected_tools=span.expected_tools,
+                )
+                api_case = create_api_test_case(
+                    test_case=llm_test_case,
+                    trace=trace_api,
+                    index=None,
+                )
+                if isinstance(api_case, LLMApiTestCase):
+                    api_case.metric_collection = span.metric_collection
+                    api_test_cases.append(api_case)
+
+            for child in span.children or []:
+                dfs(child)
+
+        for root in trace.root_spans:
+            dfs(root)
+
+    # Prepare and post TestRun using the global test run manager
+    test_run_manager = global_test_run_manager
+    test_run_manager.create_test_run(identifier=test_run_id)
+    test_run = test_run_manager.get_test_run()
+
+    for case in api_test_cases:
+        test_run.add_test_case(case)
+
+    # return test_run_manager.post_test_run(test_run) TODO: add after test run with metric collection is implemented