verifiers 0.2.2.dev71__tar.gz → 0.2.2.dev73__tar.gz

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev71
+Version: 0.2.2.dev73
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/tests/v1/conftest.py

@@ -216,20 +216,17 @@ def run_v1_server():
 
 @pytest.fixture
 async def live_ctx():
-    """A live `ModelContext` (the e2e default model + endpoint, provider-default
-    sampling) for driving `Agent` directly — the agent-surface counterpart of `run_v1`."""
-    from verifiers.v1.clients import EvalClientConfig, ModelContext, resolve_client
+    """The e2e `ModelContext` (default model + endpoint config, provider-default sampling)
+    for driving `Agent` directly — the agent-surface counterpart of `run_v1`."""
+    from verifiers.v1.clients import EvalClientConfig, ModelContext
     from verifiers.v1.types import SamplingConfig
 
-    client = resolve_client(EvalClientConfig())
-    try:
-        yield ModelContext(
-            model="deepseek/deepseek-v4-flash-0731",
-            client=client,
-            sampling=SamplingConfig(max_tokens=2048),
-        )
-    finally:
-        await client.close()
+    # Endpoint config only — each rollout builds and closes its own client.
+    yield ModelContext(
+        model="deepseek/deepseek-v4-flash-0731",
+        client=EvalClientConfig(),
+        sampling=SamplingConfig(max_tokens=2048),
+    )
 
 
 @pytest.fixture

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/tests/v1/test_e2e.py

@@ -180,8 +180,8 @@ async def test_interaction(live_ctx):
             harness=NullHarnessConfig(id="null"),
             model=live_ctx.model,
             sampling=live_ctx.sampling,
+            client=live_ctx.client,
         ),
-        client=live_ctx.client,
     )
     task = vf.Task(
         vf.TaskData(

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/verifiers/v1/agent.py

@@ -15,10 +15,8 @@ from dataclasses import dataclass
 from typing import Self
 
 from verifiers.v1.clients import (
-    Client,
     EvalClientConfig,
     ModelContext,
-    resolve_client,
 )
 from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
 from verifiers.v1.dialects import parse_message
@@ -228,10 +226,10 @@ class Interaction:
 class Agent:
     """A configured harness + model + runtime policy, runnable on any task.
 
-    Built from an `AgentConfig` alone; `client=`/`interception=` inject live
-    resources to borrow — agents on one endpoint should share one `Client`, and a
-    live `Interception`'s owner keeps its lifecycle. The config's `runtime` is a
-    *policy*: each `run` provisions a fresh box from it, resolved
+    Built from an `AgentConfig` alone; `interception=` injects a live resource to
+    borrow — its owner keeps the lifecycle. The endpoint stays config: each rollout
+    builds and closes its own `Client`, so an agent holds no transport. The config's
+    `runtime` is a *policy*: each `run` provisions a fresh box from it, resolved
     per task; `run(runtime=...)` places the run into an existing box instead
     (borrowed boxes are never started or torn down by the run)."""
 
@@ -239,7 +237,6 @@ class Agent:
         self,
         config: AgentConfig,
         *,
-        client: Client | None = None,
         interception: Interception | None = None,
     ) -> None:
         from verifiers.v1.utils.loaders import harness_config_type, load_harness
@@ -259,12 +256,9 @@ class Agent:
             config = config.model_copy(update={"sampling": Sampling()})
         self.config = config
         self.harness = load_harness(config.harness)
-        self._owns_client = client is None
-        if self._owns_client:
-            client = resolve_client(config.client or EvalClientConfig())
         self.ctx = ModelContext(
             model=config.model,
-            client=client,
+            client=config.client or EvalClientConfig(),
             sampling=config.sampling,
         )
         self._closed = False
@@ -306,22 +300,14 @@ class Agent:
                 # A failed __aenter__ gets no __aexit__ from `async with`: unwind
                 # here, or the agent stays "already entered" forever.
                 self._entered, self._server = False, None
-                if self._owns_client:
-                    self._closed = True
-                    await self.ctx.client.close()
                 raise
         return self
 
     async def __aexit__(self, *exc) -> None:
         self._entered = False
         server, self._server = self._server, None
-        try:
-            if server is not None:
-                await server.__aexit__(*exc)
-        finally:
-            if self._owns_client:
-                self._closed = True
-                await self.ctx.client.close()
+        if server is not None:
+            await server.__aexit__(*exc)
 
     def _interception_for(
         self, run_is_local: bool, task: Task, shared_tools: Mapping
@@ -584,7 +570,6 @@ class _EpisodeAgent(Agent):
         self,
         config: AgentConfig,
         *,
-        client: Client,
         interception: Interception | None,
         name: str,
         shared_tools: Mapping[str, SharedToolServer],
@@ -595,7 +580,7 @@ class _EpisodeAgent(Agent):
         on_discard: Callable[[Trace], None] | None,
         warned_resources: set,
     ) -> None:
-        super().__init__(config, client=client, interception=interception)
+        super().__init__(config, interception=interception)
         # Resource warnings dedupe env-wide, not per episode.
         self._warned_resources = warned_resources
         self._name = name
@@ -690,12 +675,11 @@ class _EpisodeAgent(Agent):
 def make_agent(
     config: AgentConfig,
     *,
-    client: Client | None = None,
     interception: Interception | None = None,
 ) -> Agent:
-    """The agent for a config; `client`/`interception` inject live resources to
-    borrow, everything else comes from the config."""
-    return Agent(config, client=client, interception=interception)
+    """The agent for a config; `interception` injects a live resource to borrow,
+    everything else comes from the config."""
+    return Agent(config, interception=interception)
 
 
 MakeAgent = Callable[[str, AgentConfig], Agent]

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/verifiers/v1/cli/eval/runner.py

@@ -14,7 +14,7 @@ from verifiers.v1.cli.output import (
     output_path,
     save_config,
 )
-from verifiers.v1.clients import ModelContext, resolve_client
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode
@@ -26,7 +26,6 @@ logger = logging.getLogger(__name__)
 
 async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
-    client = resolve_client(config.client)
     taskset = env.taskset
     if config.num_tasks is None and taskset.INFINITE:
         raise ValueError(
@@ -36,7 +35,9 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     if config.num_tasks is not None:
         selected = selected.head(config.num_tasks)
     tasks = list(selected)
-    ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
+    ctx = ModelContext(
+        client=config.client, model=config.model, sampling=config.sampling
+    )
     semaphore = (
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
     )
@@ -107,7 +108,6 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
 
                 push_state.started = True
                 await asyncio.to_thread(push_traces, episodes, config, push_state)
-    await client.close()
     return episodes
 
 

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/verifiers/v1/clients/__init__.py

@@ -1,13 +1,12 @@
-from verifiers.v1.clients.client import Client, ModelContext
-from verifiers.v1.clients.config import (
+from verifiers.v1.clients.client import Client, ModelContext, resolve_client
+from verifiers.v1.clients.eval import EvalClient
+from verifiers.v1.clients.train import TrainClient
+from verifiers.v1.configs.client import (
     BaseClientConfig,
     ClientConfig,
     EvalClientConfig,
     TrainClientConfig,
-    resolve_client,
 )
-from verifiers.v1.clients.eval import EvalClient
-from verifiers.v1.clients.train import TrainClient
 
 __all__ = [
     "BaseClientConfig",

verifiers-0.2.2.dev73/verifiers/v1/clients/base.py

@@ -0,0 +1,40 @@
+"""Shared client plumbing: transport defaults and URL/key utilities."""
+
+import re
+
+import httpx
+from openai import AsyncOpenAI
+
+from verifiers.v1.configs.client import BaseClientConfig, resolve_api_key
+
+# Mirrors the OAI SDK defaults
+DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=600.0, write=600.0, pool=600.0)
+DEFAULT_LIMITS = httpx.Limits(max_connections=1000, max_keepalive_connections=100)
+MAX_RETRIES = 0
+"""No client-side retries: failures surface to the harness SDK and the trace instead of
+being silently reattempted."""
+
+# An API version path segment (`v1`, `v2`, ...) — the only kind `join_url` dedups.
+VERSION_SEGMENT = re.compile(r"v\d+")
+
+
+def build_async_openai(config: BaseClientConfig) -> AsyncOpenAI:
+    return AsyncOpenAI(
+        base_url=config.base_url,
+        api_key=resolve_api_key(config),
+        default_headers=config.headers or None,
+        timeout=DEFAULT_TIMEOUT,
+        max_retries=MAX_RETRIES,
+        http_client=httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, limits=DEFAULT_LIMITS),
+    )
+
+
+def join_url(base_url: str, path: str) -> str:
+    """Join `base_url` with a dialect path without repeating the API version segment:
+    `.../api/v1` + `/v1/messages` -> `.../api/v1/messages`. Only version-shaped segments
+    dedup, so a base ending in `/chat` doesn't swallow `/chat/completions`."""
+    head = path.split("/")[1] if path.startswith("/") else ""
+    base = base_url.rstrip("/")
+    if VERSION_SEGMENT.fullmatch(head) and base.endswith(f"/{head}"):
+        base = base[: -len(head) - 1]
+    return base + path

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/verifiers/v1/clients/client.py

@@ -1,22 +1,21 @@
 """Client interfaces for model inference and relay."""
 
-import logging
 from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
 from dataclasses import dataclass, field
 
+from verifiers.v1.configs.client import (
+    BaseClientConfig,
+    ClientConfig,
+    TrainClientConfig,
+)
 from verifiers.v1.dialects import Dialect
 from verifiers.v1.graph import PendingTurn
 from verifiers.v1.types import Response, Sampling, SamplingConfig
 
-logger = logging.getLogger(__name__)
-
 SESSION_ID_HEADER = "X-Session-ID"
-"""Per-rollout routing header. Every turn of one rollout sends the same value (the trace id),
-so a session-affinity router (e.g. vLLM's ``consistent_hash`` policy keyed on its
-``request_id_headers``) pins all of a rollout's turns to the same engine — keeping the
-growing cross-turn prefix warm in that engine's KV cache instead of re-prefilling it
-cold on a random shard each turn."""
+"""Per-rollout routing header (the trace id, same value every turn), so a session-affinity
+router pins a rollout's turns to one engine and its growing prefix stays KV-cached."""
 
 
 @dataclass
@@ -40,14 +39,10 @@ class Client(ABC):
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> Response:
-        """Run one completion -> a vf `Response`. The eval client forwards the native JSON and
-        eligible end-to-end headers, then parses a copy via `dialect`; the train client derives
-        the typed prompt from `body` and tokenizes it.
-
-        `session_id` is the rollout's stable id (the trace id); when set, the client sends it
-        as the `SESSION_ID_HEADER` so a session-affinity router keeps the rollout's turns on
-        one engine for cross-turn prefix-cache reuse. `turn` is the graph-resolved prompt
-        prefix; train clients may use it for renderer bridging, while relay clients ignore it."""
+        """Run one completion -> a vf `Response`. The eval client forwards the native JSON
+        and parses a copy via `dialect`; the train client renders `body` to token ids.
+        `session_id` is the rollout's trace id (sent as `SESSION_ID_HEADER`); `turn` is the
+        graph-resolved prompt prefix, used by train clients for renderer bridging."""
 
     async def relay(
         self,
@@ -78,10 +73,20 @@ class Client(ABC):
         pass
 
 
+def resolve_client(config: BaseClientConfig) -> Client:
+    if isinstance(config, TrainClientConfig):
+        from verifiers.v1.clients.train import TrainClient
+
+        return TrainClient(config)
+    from verifiers.v1.clients.eval import EvalClient
+
+    return EvalClient(config)
+
+
 @dataclass(frozen=True)
 class ModelContext:
-    """Client, model, and sampling settings for one rollout."""
+    """Model, endpoint config, and sampling for one rollout."""
 
     model: str
-    client: Client
+    client: ClientConfig
     sampling: Sampling = field(default_factory=Sampling)

{verifiers-0.2.2.dev71 → verifiers-0.2.2.dev73}/verifiers/v1/clients/eval.py

@@ -1,16 +1,4 @@
-"""The eval client: relay the program's native request to the provider.
-
-`EvalClient` (the default) is a thin `httpx` forwarder: it sends the program's request body
-without a typed round-trip, mutating only what the eval owns (model + sampling, via the dialect's
-`apply_overrides`). Eligible end-to-end request headers are forwarded too; rollout auth, body
-framing, and connection headers are replaced. The provider response is parsed into a vf
-`Response` for the trace, while its full JSON object stays on `Response.raw` for the interception
-server to return.
-
-The transport is provider-agnostic: the dialect supplies the upstream path + auth headers, so a
-new wire format (incl. non-OpenAI providers like Anthropic) is just a new `Dialect` — no client
-change. Endpoint config (base url, api key, billing headers) comes from the client config.
-"""
+"""The eval client: proxies harness-native request to the provider."""
 
 import re
 from collections.abc import Mapping
@@ -19,7 +7,9 @@ import httpx
 from pydantic import ValidationError
 from pydantic_core import from_json, to_json
 
+from verifiers.v1.clients.base import DEFAULT_LIMITS, DEFAULT_TIMEOUT, join_url
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client, RelayReply
+from verifiers.v1.configs.client import BaseClientConfig, resolve_api_key
 from verifiers.v1.dialects import Dialect
 from verifiers.v1.errors import model_error
 from verifiers.v1.graph import PendingTurn
@@ -57,6 +47,8 @@ _BLOCKED_REQUEST_HEADERS = frozenset(
         "signature-input",
     }
 )
+
+
 # Atomic so one CRLF cannot backtrack into two line endings and split an event mid-field.
 _SSE_EVENT_END = re.compile(rb"(?>\r\n|\r|\n){2}")
 
@@ -64,22 +56,13 @@ _SSE_EVENT_END = re.compile(rb"(?>\r\n|\r|\n){2}")
 class EvalClient(Client):
     """Relay native JSON to the provider and parse a copy for the trace."""
 
-    def __init__(
-        self, base_url: str, api_key: str, headers: dict[str, str] | None = None
-    ) -> None:
-        self.base_url = base_url.rstrip("/")
-        self.api_key = api_key
+    def __init__(self, config: BaseClientConfig) -> None:
+        self.base_url = config.base_url
+        self.api_key = resolve_api_key(config)
         # Keep endpoint headers separate so they can override intercepted request headers before
         # the dialect's provider authentication is applied.
-        self.headers = dict(headers or {})
-        # No timeout: agentic completions are slow and the rollout timeout is the real backstop.
-        # Build full URLs ourselves (base_url + dialect.upstream_path) rather than relying on
-        # httpx base-url joining, which drops the base path for a leading-slash request path.
-        # Match V1's default concurrency while retaining HTTPX's 20-idle keepalive bound.
-        self.http = httpx.AsyncClient(
-            timeout=None,
-            limits=httpx.Limits(max_connections=128, max_keepalive_connections=20),
-        )
+        self.headers = dict(config.headers or {})
+        self.client = httpx.AsyncClient(timeout=DEFAULT_TIMEOUT, limits=DEFAULT_LIMITS)
 
     async def get_response(
         self,
@@ -92,7 +75,7 @@ class EvalClient(Client):
         headers: Mapping[str, str] | None = None,
     ) -> Response:
         resp = await self._request(
-            self.base_url + dialect.upstream_path,
+            join_url(self.base_url, dialect.upstream_path),
             dialect.apply_overrides(body, model, sampling_args),
             self._headers(dialect, headers, session_id),
         )
@@ -118,12 +101,9 @@ class EvalClient(Client):
         incoming: Mapping[str, str] | None,
         session_id: str | None,
     ) -> httpx.Headers:
-        """Build provider headers from the intercepted request.
-
-        Preserve provider feature headers such as `openai-beta` / `anthropic-beta`,
-        discard localhost auth and transport framing, then apply endpoint-configured headers,
-        session routing, and real provider auth.
-        """
+        """Provider headers from the intercepted request: keep feature headers
+        (`openai-beta`, `anthropic-beta`), discard localhost auth and framing, then apply
+        endpoint headers, session routing, and real provider auth."""
         headers = httpx.Headers(incoming)
         connection = headers.pop("connection", "")
         for name in _BLOCKED_REQUEST_HEADERS | set(
@@ -145,14 +125,14 @@ class EvalClient(Client):
         stream: bool = False,
     ) -> httpx.Response:
         headers.setdefault("content-type", "application/json")
-        request = self.http.build_request(
+        request = self.client.build_request(
             "POST",
             url,
             content=to_json(body, inf_nan_mode="null"),
             headers=headers,
         )
         try:
-            response = await self.http.send(request, stream=stream)
+            response = await self.client.send(request, stream=stream)
         except httpx.TimeoutException as e:
             raise model_error(str(e), status_code=504) from e
         except httpx.HTTPError as e:
@@ -163,9 +143,6 @@ class EvalClient(Client):
             try:
                 response.raise_for_status()
             except httpx.HTTPStatusError as e:
-                # relay the provider's status (and body) so the harness SDK retries 5xx/429 and not
-                # 4xx; an empty/HTML body (e.g. a 404 from a base_url missing `/v1`) would otherwise
-                # make an information-free ProviderError
                 raise model_error(
                     f"upstream {e.response.status_code}: {e.response.text}",
                     status_code=e.response.status_code,
@@ -193,7 +170,7 @@ class EvalClient(Client):
         # Relay complete SSE events so the interception server can safely insert keepalives
         # between them. Error responses are mapped before any event is handed back.
         resp = await self._request(
-            self.base_url + dialect.upstream_path,
+            join_url(self.base_url, dialect.upstream_path),
             dialect.apply_overrides(body, model, sampling_args),
             self._headers(dialect, headers, session_id),
             stream=True,
@@ -228,11 +205,11 @@ class EvalClient(Client):
     ) -> dict:
         # A side request (e.g. count_tokens): relay its native JSON and return the provider JSON.
         resp = await self._request(
-            self.base_url + route,
+            join_url(self.base_url, route),
             body,
             self._headers(dialect, headers, None),
         )
         return from_json(resp.content)
 
     async def close(self) -> None:
-        await self.http.aclose()
+        await self.client.aclose()