verifiers 0.3.2.dev164__tar.gz → 0.3.2.dev166__tar.gz

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev164
+Version: 0.3.2.dev166
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/tests/v1/conftest.py

@@ -156,8 +156,11 @@ def _eval_config(
         _configure_prime_runtimes(runtime_cfg)
         env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
     retries = {
-        "max_retries": 2,
-        "include": ["ProviderError", "InterceptionError", "HarnessError"],
+        "max_retries": 0,
+        "rules": [
+            {"type": error_type, "max_retries": 2}
+            for error_type in ("ProviderError", "InterceptionError", "HarnessError")
+        ],
     }
     env_cfg.setdefault("retries", retries)
     # Per-run caps live on the seats: resolve the env's declared roles and cap

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/verifiers/v1/__init__.py

@@ -21,7 +21,7 @@ from verifiers.v1.configs.cli.env import (
 from verifiers.v1.configs.env import EnvConfig, SharedEnvConfig, default_agent_harness
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig, Judges
-from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.retries import RetryConfig, RetryRule
 from verifiers.v1.configs.select import SelectCLIConfig, SelectConfig, TaskMatchConfig
 from verifiers.v1.configs.serve import (
     ElasticPoolConfig,
@@ -330,6 +330,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "default_agent_harness",
     "pool_serve_kwargs",
     "RetryConfig",
+    "RetryRule",
     # agent
     "Agent",
     "Agents",

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/verifiers/v1/agent.py

@@ -50,7 +50,7 @@ from verifiers.v1.utils.compile import (
     resolve_runtime_config,
     validate_pairing,
 )
-from verifiers.v1.utils.retries import backoff, trace_should_retry
+from verifiers.v1.utils.retries import RetryState, backoff
 
 __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 
@@ -400,12 +400,13 @@ class Agent:
         if self._closed:
             raise RuntimeError("Agent is closed; create a new agent")
         retry = self.config.retries
+        retry_state = RetryState(retry)
         history: list = []
-        for attempt in range(retry.max_retries + 1):
+        for attempt in range(retry_state.max_retries + 1):
             trace = await self._run_once(
                 task, runtime, tools, on_trace, collect_artifacts
             )
-            if attempt == retry.max_retries or not trace_should_retry(trace, retry):
+            if attempt == retry_state.max_retries or trace.ok:
                 break
             if runtime is not None:
                 logger.warning(
@@ -413,14 +414,17 @@ class Agent:
                     "longer the task's start state); the error stands"
                 )
                 break
+            cause = retry_state.next_error(trace.errors)
+            if cause is None:
+                break
             history.extend(trace.errors)
             delay = backoff(attempt)
             logger.warning(
                 "retrying agent rollout (retry %d/%d) in %.1fs after error: %s",
                 attempt + 1,
-                retry.max_retries,
+                retry_state.max_retries,
                 delay,
-                trace.last_error.type if trace.last_error else "?",
+                cause.type,
             )
             await asyncio.sleep(delay)
         if history:

verifiers-0.3.2.dev166/verifiers/v1/configs/retries.py

@@ -0,0 +1,48 @@
+"""Whole-rollout retry rules — each agent's own and the env's episode fallback."""
+
+import re
+from typing import Annotated, Literal
+
+from pydantic import Field, StrictInt, field_validator
+from pydantic_config import BaseConfig
+
+StatusCode = (
+    Annotated[StrictInt, Field(ge=100, le=599)]
+    | Literal["1xx", "2xx", "3xx", "4xx", "5xx"]
+)
+
+
+class RetryRule(BaseConfig):
+    """Match all specified fields. First matching rule wins for each error."""
+
+    type: str | None = None
+    """Exact recorded exception class name, e.g. ``ProviderError``."""
+    status_code: list[StatusCode] | None = None
+    """Match any listed HTTP status or status class; absent status never matches."""
+    message: str | None = None
+    """Regex search of the error message; plain text matches a substring."""
+    max_retries: int = Field(ge=0)
+    """Required retry budget across the run. Explicit zero excludes matching errors."""
+
+    @field_validator("message")
+    @classmethod
+    def validate_message(cls, value: str | None) -> str | None:
+        if value is not None:
+            try:
+                re.compile(value)
+            except re.error as exc:
+                raise ValueError(f"Invalid message regex: {exc}") from exc
+        return value
+
+
+class RetryConfig(BaseConfig):
+    """Ordered rules for opt-in whole-rollout retries.
+
+    Fields within a rule are ANDed. An exhausted rule never falls through to a
+    later rule for that error; another captured error can still trigger a retry.
+    """
+
+    max_retries: int = Field(0, ge=0)
+    """Default budget for errors that match no rule. Matching rules override it."""
+    rules: list[RetryRule] = Field(default_factory=list)
+    """Ordered overrides; a matching rule may enable retries or deny them with zero."""

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/verifiers/v1/dialects/responses.py

@@ -393,6 +393,21 @@ class ResponsesDialect(Dialect[OpenAIResponse]):
                     safe_input.append(item)
                     continue
                 kind = item.get("type")
+                if (
+                    kind == "reasoning"
+                    and not item.get("encrypted_content")
+                    and isinstance(content := item.get("content"), list)
+                    and content
+                    and all(
+                        isinstance(part, dict)
+                        and part.get("type") == "reasoning_text"
+                        and isinstance(part.get("text"), str)
+                        for part in content
+                    )
+                    and any(part["text"] for part in content)
+                ):
+                    # Inline reasoning needs no provider-side history lookup.
+                    item.pop("id", None)
                 if kind in ("additional_tools", "tool_search_output"):
                     if blocked := request_filter.blocked(
                         {**item, "tools": []}, item_path

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/verifiers/v1/env.py

@@ -43,7 +43,10 @@ def _as_error(e: Exception) -> Error:
     """`e` as an episode-level `Error`. Call inside the `except` handling `e` — the
     traceback comes from the active exception context."""
     return Error(
-        type=type(e).__name__, message=str(e), traceback=traceback.format_exc()
+        type=type(e).__name__,
+        message=str(e),
+        status_code=getattr(e, "status_code", None),
+        traceback=traceback.format_exc(),
     )
 
 

{verifiers-0.3.2.dev164 → verifiers-0.3.2.dev166}/verifiers/v1/utils/retries.py

@@ -6,7 +6,7 @@ Two opt-in whole-run retry atoms sit above that: `Agent.run` reruns ITS OWN roll
 while the trace ends with a retryable error (`--env.<agent>.retries` — a flaky
 grader retries without re-burning the solver), and `run_episode_with_retry` reruns
 the entire episode (`--env.retries`) — the coarse fallback for faults no agent
-owns: the env's own hooks, cross-agent state. Both match by exception type name;
+owns: the env's own hooks, cross-agent state. Both use ordered retry rules;
 both off by default.
 """
 
@@ -15,7 +15,8 @@ from __future__ import annotations
 import asyncio
 import logging
 import random
-from collections.abc import Awaitable, Callable
+import re
+from collections.abc import Awaitable, Callable, Iterable
 from typing import TYPE_CHECKING
 
 from tenacity import (
@@ -27,7 +28,7 @@ from tenacity import (
     wait_exponential_jitter,
 )
 
-from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.retries import RetryConfig, RetryRule
 
 if TYPE_CHECKING:
     from verifiers.v1.episode import Episode
@@ -73,35 +74,47 @@ def retrying(
     )
 
 
-def _retryable(error: Error | None, retry: RetryConfig) -> bool:
-    """Whether `error` matches the retry policy: its exception type is included (and
-    not excluded)."""
-    if error is None:
-        return False
-    if error.type in retry.exclude:
-        return False
-    if retry.include:
-        return error.type in retry.include
-    return True
-
-
-def trace_should_retry(trace, retry: RetryConfig) -> bool:
-    """Whether a finished agent rollout should be retried: any captured error on
-    its trace is retryable (all captures count, not just the most recent)."""
-    return any(_retryable(e, retry) for e in trace.errors)
-
-
-def episode_should_retry(episode: Episode, retry: RetryConfig) -> bool:
-    """Whether a finished episode should be retried: any LIVE captured error —
-    episode-level, or on a trace that ended not-`ok` — is retryable. An `ok` trace's
-    errors are history its own per-agent retry already recovered from, never
-    grounds to re-run the episode. All of a failed trace's captures count, not just
-    the most recent: a retryable failure followed by a teardown error would
-    otherwise never retry. Episode-atomic — a half-played sibling context isn't
-    reproducible."""
-    return any(_retryable(e, retry) for e in episode.errors) or any(
-        _retryable(e, retry) for t in episode.traces if not t.ok for e in t.errors
-    )
+class RetryState:
+    """Per-run budgets, shared across attempts but never across concurrent runs."""
+
+    def __init__(self, config: RetryConfig) -> None:
+        self.rules = [*config.rules, RetryRule(max_retries=config.max_retries)]
+        # Each retry consumes one finite budget, so their sum bounds the run.
+        self.max_retries = sum(rule.max_retries for rule in self.rules)
+        self.used = [0] * len(self.rules)
+        self.patterns = [
+            re.compile(rule.message) if rule.message is not None else None
+            for rule in self.rules
+        ]
+
+    def next_error(self, errors: Iterable[Error]) -> Error | None:
+        """Consume one retry for the first eligible error in capture order.
+
+        A denied/exhausted first match shadows later rules for that error only.
+        Return the triggering error so logs identify the actual retry cause.
+        """
+        for error in errors:
+            for index, rule in enumerate(self.rules):
+                if rule.type is not None and error.type != rule.type:
+                    continue
+                if rule.status_code is not None and (
+                    error.status_code is None
+                    or not any(
+                        error.status_code == status
+                        if isinstance(status, int)
+                        else error.status_code // 100 == int(status[0])
+                        for status in rule.status_code
+                    )
+                ):
+                    continue
+                pattern = self.patterns[index]
+                if pattern is not None and pattern.search(error.message) is None:
+                    continue
+                if self.used[index] < rule.max_retries:
+                    self.used[index] += 1
+                    return error
+                break
+        return None
 
 
 async def run_episode_with_retry(
@@ -113,13 +126,17 @@ async def run_episode_with_retry(
     attempts' errors are prepended so the episode shows the full history; a final
     good attempt returns clean."""
     history: list = []
-    for attempt in range(retry.max_retries + 1):
+    state = RetryState(retry)
+    for attempt in range(state.max_retries + 1):
         final = await run()
-        if attempt == retry.max_retries or not episode_should_retry(final, retry):
+        if attempt == state.max_retries or final.ok:
+            break
+        # Successful traces may contain recovered failures from agent retries.
+        errors = list(final.errors)
+        errors.extend(e for trace in final.traces if not trace.ok for e in trace.errors)
+        cause = state.next_error(errors)
+        if cause is None:
             break
-        cause = final.last_error or next(
-            (t.last_error for t in final.traces if t.last_error), None
-        )
         history.extend(final.errors)
         for trace in final.traces:
             history.extend(trace.errors)
@@ -128,9 +145,9 @@ async def run_episode_with_retry(
             "retrying episode %s (retry %d/%d) in %.1fs after error: %s",
             final.id,
             attempt + 1,
-            retry.max_retries,
+            state.max_retries,
             delay,
-            cause.type if cause else "?",
+            cause.type,
         )
         await asyncio.sleep(delay)
     if history:

verifiers-0.3.2.dev164/verifiers/v1/configs/retries.py

@@ -1,17 +0,0 @@
-"""Whole-rollout retry policy — each agent's own and the env's episode fallback."""
-
-from pydantic import Field
-from pydantic_config import BaseConfig
-
-
-class RetryConfig(BaseConfig):
-    """Retry a whole rollout when it ends with a captured error. `include`/`exclude`
-    name exception classes (e.g. ``ProviderError``, ``SandboxError``)."""
-
-    max_retries: int = Field(0, ge=0)
-    """Whole-rollout retries beyond the first attempt. Off by default — the SDKs
-    already retry transient per-call faults; rerunning a whole trajectory is opt-in."""
-    include: list[str] = Field(default_factory=list)
-    """Only retry errors whose type is listed. Empty = retry anything not excluded."""
-    exclude: list[str] = Field(default_factory=list)
-    """Never retry errors whose type is listed (wins over `include`)."""