verifiers 0.3.2.dev163__tar.gz → 0.3.2.dev165__tar.gz

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev163
+Version: 0.3.2.dev165
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/tests/v1/conftest.py

@@ -156,8 +156,11 @@ def _eval_config(
         _configure_prime_runtimes(runtime_cfg)
         env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
     retries = {
-        "max_retries": 2,
-        "include": ["ProviderError", "InterceptionError", "HarnessError"],
+        "max_retries": 0,
+        "rules": [
+            {"type": error_type, "max_retries": 2}
+            for error_type in ("ProviderError", "InterceptionError", "HarnessError")
+        ],
     }
     env_cfg.setdefault("retries", retries)
     # Per-run caps live on the seats: resolve the env's declared roles and cap

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/__init__.py

@@ -21,7 +21,7 @@ from verifiers.v1.configs.cli.env import (
 from verifiers.v1.configs.env import EnvConfig, SharedEnvConfig, default_agent_harness
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig, Judges
-from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.retries import RetryConfig, RetryRule
 from verifiers.v1.configs.select import SelectCLIConfig, SelectConfig, TaskMatchConfig
 from verifiers.v1.configs.serve import (
     ElasticPoolConfig,
@@ -330,6 +330,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "default_agent_harness",
     "pool_serve_kwargs",
     "RetryConfig",
+    "RetryRule",
     # agent
     "Agent",
     "Agents",

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/agent.py

@@ -50,7 +50,7 @@ from verifiers.v1.utils.compile import (
     resolve_runtime_config,
     validate_pairing,
 )
-from verifiers.v1.utils.retries import backoff, trace_should_retry
+from verifiers.v1.utils.retries import RetryState, backoff
 
 __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 
@@ -400,12 +400,13 @@ class Agent:
         if self._closed:
             raise RuntimeError("Agent is closed; create a new agent")
         retry = self.config.retries
+        retry_state = RetryState(retry)
         history: list = []
-        for attempt in range(retry.max_retries + 1):
+        for attempt in range(retry_state.max_retries + 1):
             trace = await self._run_once(
                 task, runtime, tools, on_trace, collect_artifacts
             )
-            if attempt == retry.max_retries or not trace_should_retry(trace, retry):
+            if attempt == retry_state.max_retries or trace.ok:
                 break
             if runtime is not None:
                 logger.warning(
@@ -413,14 +414,17 @@ class Agent:
                     "longer the task's start state); the error stands"
                 )
                 break
+            cause = retry_state.next_error(trace.errors)
+            if cause is None:
+                break
             history.extend(trace.errors)
             delay = backoff(attempt)
             logger.warning(
                 "retrying agent rollout (retry %d/%d) in %.1fs after error: %s",
                 attempt + 1,
-                retry.max_retries,
+                retry_state.max_retries,
                 delay,
-                trace.last_error.type if trace.last_error else "?",
+                cause.type,
             )
             await asyncio.sleep(delay)
         if history:

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/clients/base.py

@@ -7,9 +7,6 @@ from openai import AsyncOpenAI
 
 from verifiers.v1.configs.client import BaseClientConfig, resolve_api_key
 
-# No read timeout: agentic completions are slow and the rollout timeout is the real
-# backstop. The connect bound stays so an unreachable endpoint still fails fast.
-DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=None, pool=None)
 DEFAULT_LIMITS = httpx.Limits(max_connections=1000, max_keepalive_connections=100)
 MAX_RETRIES = 0
 """No client-side retries: failures surface to the harness SDK and the trace instead of
@@ -19,14 +16,20 @@ being silently reattempted."""
 VERSION_SEGMENT = re.compile(r"v\d+")
 
 
+def build_async_httpx(config: BaseClientConfig) -> httpx.AsyncClient:
+    return httpx.AsyncClient(
+        timeout=httpx.Timeout(**config.timeout.model_dump()), limits=DEFAULT_LIMITS
+    )
+
+
 def build_async_openai(config: BaseClientConfig) -> AsyncOpenAI:
+    # The SDK adopts the http client's timeout, so it is configured in one place.
     return AsyncOpenAI(
         base_url=config.base_url,
         api_key=resolve_api_key(config),
         default_headers=config.headers or None,
-        timeout=DEFAULT_TIMEOUT,
         max_retries=MAX_RETRIES,
-        http_client=httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, limits=DEFAULT_LIMITS),
+        http_client=build_async_httpx(config),
     )
 
 

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/clients/eval.py

@@ -7,7 +7,7 @@ import httpx
 from pydantic import ValidationError
 from pydantic_core import from_json, to_json
 
-from verifiers.v1.clients.base import DEFAULT_LIMITS, DEFAULT_TIMEOUT, join_url
+from verifiers.v1.clients.base import build_async_httpx, join_url
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client, RelayReply
 from verifiers.v1.configs.client import BaseClientConfig, resolve_api_key
 from verifiers.v1.dialects import Dialect
@@ -68,7 +68,7 @@ class EvalClient(Client):
         # Keep endpoint headers separate so they can override intercepted request headers before
         # the dialect's provider authentication is applied.
         self.headers = dict(config.headers or {})
-        self.client = httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, limits=DEFAULT_LIMITS)
+        self.client = build_async_httpx(config)
 
     async def get_response(
         self,

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/configs/client.py

@@ -26,6 +26,21 @@ PRIME_INFERENCE_HOST = "pinference.ai"
 PRIME_TEAM_ID_HEADER = "X-Prime-Team-ID"
 
 
+class ClientTimeoutConfig(BaseConfig):
+    """Timeouts (in seconds) for each phase of an HTTP request to the endpoint; `None`
+    waits indefinitely. Distinct from the agent's stage timeouts, which bound a rollout."""
+
+    connect: float | None = Field(30.0, gt=0)
+    """Time to establish a connection, so an unreachable endpoint still fails fast."""
+    read: float | None = Field(None, gt=0)
+    """Time between received bytes. Unbounded by default: agentic completions are slow
+    and the rollout timeout is the real backstop."""
+    write: float | None = Field(None, gt=0)
+    """Time between sent bytes."""
+    pool: float | None = Field(None, gt=0)
+    """Time to acquire a connection from the pool."""
+
+
 class BaseClientConfig(BaseConfig):
     """An OpenAI-compatible endpoint. The API key is read from an env var."""
 
@@ -33,6 +48,7 @@ class BaseClientConfig(BaseConfig):
     api_key_var: str = "PRIME_API_KEY"
     headers: dict[str, str] = Field(default_factory=dict)
     """Extra HTTP headers sent on every request."""
+    timeout: ClientTimeoutConfig = ClientTimeoutConfig()
 
     @model_validator(mode="after")
     def apply_prime_config(self) -> "BaseClientConfig":
@@ -65,7 +81,7 @@ class EvalClientConfig(BaseClientConfig):
 
 class TrainClientConfig(BaseClientConfig):
     """Training: a vLLM `/inference/v1/generate` endpoint with client-side tokenization (via
-    `TrainClient`), so responses carry token ids + logprobs. Needs a running vLLM engine."""
+    `TrainClient`), so responses carry token IDs and logprobs. Needs a running vLLM engine."""
 
     type: Literal["train"] = "train"
     renderer: RendererConfig | None = None

verifiers-0.3.2.dev165/verifiers/v1/configs/retries.py

@@ -0,0 +1,48 @@
+"""Whole-rollout retry rules — each agent's own and the env's episode fallback."""
+
+import re
+from typing import Annotated, Literal
+
+from pydantic import Field, StrictInt, field_validator
+from pydantic_config import BaseConfig
+
+StatusCode = (
+    Annotated[StrictInt, Field(ge=100, le=599)]
+    | Literal["1xx", "2xx", "3xx", "4xx", "5xx"]
+)
+
+
+class RetryRule(BaseConfig):
+    """Match all specified fields. First matching rule wins for each error."""
+
+    type: str | None = None
+    """Exact recorded exception class name, e.g. ``ProviderError``."""
+    status_code: list[StatusCode] | None = None
+    """Match any listed HTTP status or status class; absent status never matches."""
+    message: str | None = None
+    """Regex search of the error message; plain text matches a substring."""
+    max_retries: int = Field(ge=0)
+    """Required retry budget across the run. Explicit zero excludes matching errors."""
+
+    @field_validator("message")
+    @classmethod
+    def validate_message(cls, value: str | None) -> str | None:
+        if value is not None:
+            try:
+                re.compile(value)
+            except re.error as exc:
+                raise ValueError(f"Invalid message regex: {exc}") from exc
+        return value
+
+
+class RetryConfig(BaseConfig):
+    """Ordered rules for opt-in whole-rollout retries.
+
+    Fields within a rule are ANDed. An exhausted rule never falls through to a
+    later rule for that error; another captured error can still trigger a retry.
+    """
+
+    max_retries: int = Field(0, ge=0)
+    """Default budget for errors that match no rule. Matching rules override it."""
+    rules: list[RetryRule] = Field(default_factory=list)
+    """Ordered overrides; a matching rule may enable retries or deny them with zero."""

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/env.py

@@ -43,7 +43,10 @@ def _as_error(e: Exception) -> Error:
     """`e` as an episode-level `Error`. Call inside the `except` handling `e` — the
     traceback comes from the active exception context."""
     return Error(
-        type=type(e).__name__, message=str(e), traceback=traceback.format_exc()
+        type=type(e).__name__,
+        message=str(e),
+        status_code=getattr(e, "status_code", None),
+        traceback=traceback.format_exc(),
     )
 
 

{verifiers-0.3.2.dev163 → verifiers-0.3.2.dev165}/verifiers/v1/utils/retries.py

@@ -6,7 +6,7 @@ Two opt-in whole-run retry atoms sit above that: `Agent.run` reruns ITS OWN roll
 while the trace ends with a retryable error (`--env.<agent>.retries` — a flaky
 grader retries without re-burning the solver), and `run_episode_with_retry` reruns
 the entire episode (`--env.retries`) — the coarse fallback for faults no agent
-owns: the env's own hooks, cross-agent state. Both match by exception type name;
+owns: the env's own hooks, cross-agent state. Both use ordered retry rules;
 both off by default.
 """
 
@@ -15,7 +15,8 @@ from __future__ import annotations
 import asyncio
 import logging
 import random
-from collections.abc import Awaitable, Callable
+import re
+from collections.abc import Awaitable, Callable, Iterable
 from typing import TYPE_CHECKING
 
 from tenacity import (
@@ -27,7 +28,7 @@ from tenacity import (
     wait_exponential_jitter,
 )
 
-from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.retries import RetryConfig, RetryRule
 
 if TYPE_CHECKING:
     from verifiers.v1.episode import Episode
@@ -73,35 +74,47 @@ def retrying(
     )
 
 
-def _retryable(error: Error | None, retry: RetryConfig) -> bool:
-    """Whether `error` matches the retry policy: its exception type is included (and
-    not excluded)."""
-    if error is None:
-        return False
-    if error.type in retry.exclude:
-        return False
-    if retry.include:
-        return error.type in retry.include
-    return True
-
-
-def trace_should_retry(trace, retry: RetryConfig) -> bool:
-    """Whether a finished agent rollout should be retried: any captured error on
-    its trace is retryable (all captures count, not just the most recent)."""
-    return any(_retryable(e, retry) for e in trace.errors)
-
-
-def episode_should_retry(episode: Episode, retry: RetryConfig) -> bool:
-    """Whether a finished episode should be retried: any LIVE captured error —
-    episode-level, or on a trace that ended not-`ok` — is retryable. An `ok` trace's
-    errors are history its own per-agent retry already recovered from, never
-    grounds to re-run the episode. All of a failed trace's captures count, not just
-    the most recent: a retryable failure followed by a teardown error would
-    otherwise never retry. Episode-atomic — a half-played sibling context isn't
-    reproducible."""
-    return any(_retryable(e, retry) for e in episode.errors) or any(
-        _retryable(e, retry) for t in episode.traces if not t.ok for e in t.errors
-    )
+class RetryState:
+    """Per-run budgets, shared across attempts but never across concurrent runs."""
+
+    def __init__(self, config: RetryConfig) -> None:
+        self.rules = [*config.rules, RetryRule(max_retries=config.max_retries)]
+        # Each retry consumes one finite budget, so their sum bounds the run.
+        self.max_retries = sum(rule.max_retries for rule in self.rules)
+        self.used = [0] * len(self.rules)
+        self.patterns = [
+            re.compile(rule.message) if rule.message is not None else None
+            for rule in self.rules
+        ]
+
+    def next_error(self, errors: Iterable[Error]) -> Error | None:
+        """Consume one retry for the first eligible error in capture order.
+
+        A denied/exhausted first match shadows later rules for that error only.
+        Return the triggering error so logs identify the actual retry cause.
+        """
+        for error in errors:
+            for index, rule in enumerate(self.rules):
+                if rule.type is not None and error.type != rule.type:
+                    continue
+                if rule.status_code is not None and (
+                    error.status_code is None
+                    or not any(
+                        error.status_code == status
+                        if isinstance(status, int)
+                        else error.status_code // 100 == int(status[0])
+                        for status in rule.status_code
+                    )
+                ):
+                    continue
+                pattern = self.patterns[index]
+                if pattern is not None and pattern.search(error.message) is None:
+                    continue
+                if self.used[index] < rule.max_retries:
+                    self.used[index] += 1
+                    return error
+                break
+        return None
 
 
 async def run_episode_with_retry(
@@ -113,13 +126,17 @@ async def run_episode_with_retry(
     attempts' errors are prepended so the episode shows the full history; a final
     good attempt returns clean."""
     history: list = []
-    for attempt in range(retry.max_retries + 1):
+    state = RetryState(retry)
+    for attempt in range(state.max_retries + 1):
         final = await run()
-        if attempt == retry.max_retries or not episode_should_retry(final, retry):
+        if attempt == state.max_retries or final.ok:
+            break
+        # Successful traces may contain recovered failures from agent retries.
+        errors = list(final.errors)
+        errors.extend(e for trace in final.traces if not trace.ok for e in trace.errors)
+        cause = state.next_error(errors)
+        if cause is None:
             break
-        cause = final.last_error or next(
-            (t.last_error for t in final.traces if t.last_error), None
-        )
         history.extend(final.errors)
         for trace in final.traces:
             history.extend(trace.errors)
@@ -128,9 +145,9 @@ async def run_episode_with_retry(
             "retrying episode %s (retry %d/%d) in %.1fs after error: %s",
             final.id,
             attempt + 1,
-            retry.max_retries,
+            state.max_retries,
             delay,
-            cause.type if cause else "?",
+            cause.type,
         )
         await asyncio.sleep(delay)
     if history:

verifiers-0.3.2.dev163/verifiers/v1/configs/retries.py

@@ -1,17 +0,0 @@
-"""Whole-rollout retry policy — each agent's own and the env's episode fallback."""
-
-from pydantic import Field
-from pydantic_config import BaseConfig
-
-
-class RetryConfig(BaseConfig):
-    """Retry a whole rollout when it ends with a captured error. `include`/`exclude`
-    name exception classes (e.g. ``ProviderError``, ``SandboxError``)."""
-
-    max_retries: int = Field(0, ge=0)
-    """Whole-rollout retries beyond the first attempt. Off by default — the SDKs
-    already retry transient per-call faults; rerunning a whole trajectory is opt-in."""
-    include: list[str] = Field(default_factory=list)
-    """Only retry errors whose type is listed. Empty = retry anything not excluded."""
-    exclude: list[str] = Field(default_factory=list)
-    """Never retry errors whose type is listed (wins over `include`)."""