verifiers 0.3.2.dev142__tar.gz → 0.3.2.dev144__tar.gz

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev142
+Version: 0.3.2.dev144
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/__init__.py

@@ -14,8 +14,12 @@ from verifiers.v1.clients import (
     resolve_client,
 )
 from verifiers.v1.configs.agent import AgentConfig
-from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
-from verifiers.v1.configs.env import EnvConfig, default_agent_harness
+from verifiers.v1.configs.cli.env import (
+    merge_env_defaults,
+    narrowed_env_annotation,
+    resolve_env_field,
+)
+from verifiers.v1.configs.env import EnvConfig, SharedEnvConfig, default_agent_harness
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig, Judges
 from verifiers.v1.configs.retries import RetryConfig
@@ -30,7 +34,7 @@ from verifiers.v1.configs.task import (
     RewardFunctionConfig,
     TaskConfig,
 )
-from verifiers.v1.configs.taskset import TasksetConfig
+from verifiers.v1.configs.taskset import SharedTasksetConfig, TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
 from verifiers.v1.episode import (
@@ -286,6 +290,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Taskset",
     "TaskConfig",
     "TasksetConfig",
+    "SharedTasksetConfig",
     "DecoratedFunctionConfig",
     "RewardFunctionConfig",
     "BaseConfig",
@@ -309,8 +314,10 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Env",
     "SingleAgentEnv",
     "EnvConfig",
+    "SharedEnvConfig",
     "ServeConfig",
     "resolve_env_field",
+    "merge_env_defaults",
     "narrowed_env_annotation",
     "SingleAgentEnvConfig",
     "AgentConfig",

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/configs/cli/env.py

@@ -1,5 +1,5 @@
 """Run-config plumbing around the `[env]` block: narrowing the `env` field of
-every config that owns one.
+every config that owns one, and layering shared env defaults under it.
 
 A run composes the blocks it needs — `[env]` (what runs, `configs/env.py`),
 `[serve]` (how it's hosted, `configs/serve.py`) — plus its own fields. Nothing here is a base class: the eval
@@ -10,11 +10,11 @@ single_agent_env_config)`. The `SerializeAsAny` is load-bearing: pydantic serial
 by declared type, so a plain `EnvConfig` silently drops a narrowed subclass's agents
 and knobs from `model_dump()` — the env-server wire's payload."""
 
-from pydantic import ValidationError
+from pydantic import BaseModel, ValidationError
 from pydantic_config import BaseConfig
 
-from verifiers.v1.configs.env import EnvConfig
-from verifiers.v1.utils.generic import prefix_validation_error
+from verifiers.v1.configs.env import EnvConfig, SharedEnvConfig
+from verifiers.v1.utils.generic import deep_merge, prefix_validation_error
 
 
 def resolve_env_field(data: dict, narrowed: "type[EnvConfig] | None" = None) -> dict:
@@ -45,6 +45,32 @@ def resolve_env_field(data: dict, narrowed: "type[EnvConfig] | None" = None) ->
     return data
 
 
+def merge_env_defaults(defaults: SharedEnvConfig, env: dict | None) -> dict:
+    """A raw `env` block over `defaults`, the knobs that several envs of one run share
+    (e.g. the retries of every eval source). Only the fields set in `defaults` apply,
+    and the env's own values win. A subtree whose `id`/`type` differs from the
+    default's is the env's alone, so one plugin's knobs never leak into another's."""
+    return deep_merge(_dump_set(defaults), env or {})
+
+
+def _dump_set(config: BaseModel) -> dict:
+    """`config`'s set fields, including those set on a sub-config after construction
+    (which leaves the parent's field unmarked). Each dumped sub-config keeps its
+    `id`/`type` even at the default: `deep_merge` detects a plugin switch only when
+    both sides name it."""
+    dump = {}
+    for name in type(config).model_fields:
+        value = getattr(config, name)
+        if isinstance(value, BaseModel):
+            nested = _dump_set(value)
+            if nested or name in config.model_fields_set:
+                keys = [k for k in ("id", "type") if k in type(value).model_fields]
+                dump[name] = {**{k: getattr(value, k) for k in keys}, **nested}
+        elif name in config.model_fields_set:
+            dump[name] = config.model_dump(include={name})[name]
+    return dump
+
+
 def narrowed_env_annotation(cls) -> "type[EnvConfig] | None":
     """The env field's annotation when the CLI pre-narrowed it (`narrow_config`
     swaps in a concrete subclass). The base declaration reads as `EnvConfig` itself

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/configs/env.py

@@ -1,5 +1,6 @@
 """An environment's config — the run's single `[env]` block: the seed taskset,
-each role as an `AgentConfig` field, and the env-level knobs."""
+each role as an `AgentConfig` field, and the env-level knobs. `SharedEnvConfig` is
+the part every env has."""
 
 from typing import get_args
 
@@ -13,7 +14,7 @@ from verifiers.v1.configs.agent import (
 )
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
-from verifiers.v1.configs.taskset import TasksetConfig
+from verifiers.v1.configs.taskset import SharedTasksetConfig, TasksetConfig
 from verifiers.v1.interception import ElasticInterceptionPoolConfig, InterceptionConfig
 from verifiers.v1.types import ID
 
@@ -34,18 +35,12 @@ def _mentions_agent_config(annotation) -> bool:
     return any(_mentions_agent_config(arg) for arg in get_args(annotation))
 
 
-class EnvConfig(BaseConfig):
-    """An environment's config — the run's single `[env]` block, one subclass per
-    `Env` class (bound via `Env[YourConfig]`): each role is an
-    `AgentConfig` field with a default instance, plus env-level knobs. The run's `env`
-    field narrows to it by env `id` (else taskset id) — `--env.<role>.model` addressing."""
+class SharedEnvConfig(BaseConfig):
+    """The env knobs every env has, whatever its id and agents — what several envs of
+    one run can share. `EnvConfig` adds the env `id`, the taskset `id`, and the agents."""
 
-    id: ID = ""
-    """Which `Env` runs. Empty = the taskset's own, else `SingleAgentEnv`; set
-    to pair a reusable env with any taskset (an explicit id wins over the bundled)."""
-    # SerializeAsAny: the env-server wire needs the resolved subclass's fields.
-    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
-    """The seed taskset — the rows every rollout starts from (`--env.taskset.id`)."""
+    taskset: SharedTasksetConfig = SharedTasksetConfig()
+    """The taskset knobs every taskset has."""
     timeout: TimeoutConfig = TimeoutConfig()
     retries: RetryConfig = RetryConfig()
     """Whole-EPISODE retries — the coarse fallback for faults no agent owns; a
@@ -63,6 +58,20 @@ class EnvConfig(BaseConfig):
     interception: InterceptionConfig = ElasticInterceptionPoolConfig()
     """The interception shape: `elastic` (default), `server`, or `static`."""
 
+
+class EnvConfig(SharedEnvConfig):
+    """An environment's config — the run's single `[env]` block, one subclass per
+    `Env` class (bound via `Env[YourConfig]`): each role is an
+    `AgentConfig` field with a default instance, plus env-level knobs. The run's `env`
+    field narrows to it by env `id` (else taskset id) — `--env.<role>.model` addressing."""
+
+    id: ID = ""
+    """Which `Env` runs. Empty = the taskset's own, else `SingleAgentEnv`; set
+    to pair a reusable env with any taskset (an explicit id wins over the bundled)."""
+    # SerializeAsAny: the env-server wire needs the resolved subclass's fields.
+    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    """The seed taskset — the rows every rollout starts from (`--env.taskset.id`)."""
+
     @property
     def env_id(self) -> str:
         """The taskset id, prefixed by the paired env id (`best-of-n+gsm8k-v1`)."""

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/configs/taskset.py

@@ -9,16 +9,22 @@ from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.types import ID
 
 
-class TasksetConfig(BaseConfig):
-    id: ID = ""
-    """Installed taskset package, set with `--env.taskset.id` (or the
-    positional `eval <taskset-id>`)."""
+class SharedTasksetConfig(BaseConfig):
+    """The knobs every taskset has, whatever its id — what several tasksets of one run
+    can share."""
+
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
     system_prompt: Path | None = None
     """File whose text overrides each task's `TaskData.system_prompt` on
     iteration (e.g. a GEPA `best_system_prompt.txt`)."""
 
+
+class TasksetConfig(SharedTasksetConfig):
+    id: ID = ""
+    """Installed taskset package, set with `--env.taskset.id` (or the
+    positional `eval <taskset-id>`)."""
+
     @property
     def name(self) -> str:
         return self.id

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/env.py

@@ -272,6 +272,7 @@ class Env(ABC, Generic[ConfigT]):
                     f"{type(self).__name__}.run() exceeded its "
                     f"{self.config.timeout.episode:g}s deadline (--env.timeout.episode)"
                 )
+                episode.is_timeout = True
             episode.errors.append(_as_error(e))
             # The completed subset is the crash-safe episode; ok stays False.
             return episode
@@ -286,6 +287,7 @@ class Env(ABC, Generic[ConfigT]):
                     f"{type(self).__name__}.finalize() exceeded its "
                     f"{self.config.timeout.finalize:g}s deadline (--env.timeout.finalize)"
                 )
+                episode.is_timeout = True
             episode.errors.append(_as_error(e))
             return episode
         # Both hooks and every trace concluded — stamp the attempt's verdict

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/episode.py

@@ -99,6 +99,9 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     """The run this episode belongs to, consumer-stamped."""
     ok: bool = False
     """Whether the episode completed successfully."""
+    is_timeout: bool = False
+    """Whether the env's own `run()`/`finalize()` deadline expired; a per-agent
+    stage deadline flags its trace (`Trace.is_timeout`)."""
     errors: list[Error] = Field(default_factory=list)
     """Every error captured across attempts, oldest to newest."""
     traces: list[Trace[DataT, StateT, AgentConfigT]] = Field(default_factory=list)

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/errors.py

@@ -77,8 +77,9 @@ async def boundary(error_cls: type[RolloutError], what: str) -> AsyncIterator[No
     """Run a framework→code boundary, attributing any error escaping it to `error_cls`. An
     already-typed `RolloutError` passes through unchanged — it crossed a more specific boundary
     first (e.g. a `SandboxError` from `runtime.run` inside a reward stays a `SandboxError`). A
-    `TimeoutError` (the stage exceeded its budget) becomes `error_cls` too. `what` names the
-    boundary in the error message."""
+    `TimeoutError` the code raises itself (its own I/O) becomes `error_cls` too; a stage
+    deadline wraps the boundary from outside, so its expiry is a timeout, not an error.
+    `what` names the boundary in the error message."""
     try:
         yield
     except RolloutError:

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/rollout.py

@@ -174,6 +174,12 @@ class Rollout:
         self._failure = error
         self.trace.record_error(error)
 
+    def timeout(self, stage: str) -> None:
+        """Record `stage`'s expired deadline as this rollout's outcome: a stop, not
+        a failure — no further segments run, but finalize and scoring still do."""
+        logger.info("rollout %s: %s timeout", self.trace.id, stage)
+        self.trace.record_timeout(stage)
+
     async def open(self) -> bool:
         """Boot the rollout's world up to the point where segments can run: start
         (or borrow) the runtime, run task + harness setup, bring up the
@@ -202,6 +208,8 @@ class Rollout:
             self.harness.config.name,
             self.runtime_config.type,
         )
+        loop = asyncio.get_running_loop()
+        setup_deadline: float | None = None
         try:
             runtime_env = dict(self.task.runtime_env())
             if self._borrowed_runtime is None:
@@ -226,16 +234,16 @@ class Rollout:
             setup_deadline = (
                 None
                 if self._timeouts.setup is None
-                else asyncio.get_running_loop().time() + self._timeouts.setup
+                else loop.time() + self._timeouts.setup
             )
             async with (
-                boundary(TaskError, "task setup"),
                 asyncio.timeout_at(setup_deadline),
+                boundary(TaskError, "task setup"),
             ):
                 await invoke(self.task.setup, {"trace": self.trace, "runtime": runtime})
             async with (
-                boundary(HarnessError, "harness setup"),
                 asyncio.timeout_at(setup_deadline),
+                boundary(HarnessError, "harness setup"),
             ):
                 await self.harness.setup(runtime)
             async with boundary(ToolsetError, "building tool servers"):
@@ -273,8 +281,8 @@ class Rollout:
             # execution policy while preserving the framework routes the agent uses.
             await runtime.prepare_execution([self._endpoint, *self._urls.values()])
             async with (
-                boundary(HarnessError, "opening harness session"),
                 asyncio.timeout_at(setup_deadline),
+                boundary(HarnessError, "opening harness session"),
             ):
                 harness_data = self.trace.task.data
                 if (
@@ -340,6 +348,14 @@ class Rollout:
                         harness_data,
                         **session_kwargs,
                     )
+        except TimeoutError as e:
+            # The setup deadline's own expiry is a timeout; a TimeoutError with no
+            # expired deadline (a runtime's own I/O) stays the raw failure.
+            if setup_deadline is not None and loop.time() >= setup_deadline:
+                self.timeout("setup")
+            else:
+                self.fail(e)
+            return False
         except Exception as e:  # noqa: BLE001 - setup boundary records every rollout failure
             self.fail(e)
             return False
@@ -390,16 +406,11 @@ class Rollout:
                         return False
                 await self._harness_session.turn(messages)
         except TimeoutError as e:
-            # An expired rollout deadline is the agent breaking its time budget —
-            # an agent failure, never a clean stop. A TimeoutError from the
-            # harness's own I/O with no expired deadline stays the raw failure.
+            # An expired rollout deadline is the agent breaking its time budget: a
+            # timeout stop. A TimeoutError from the harness's own I/O with no
+            # expired deadline stays the raw failure.
             if self.deadline_at is not None and (loop.time() >= self.deadline_at):
-                self.fail(
-                    HarnessError(
-                        f"agent timeout: rollout exceeded its "
-                        f"{self._timeouts.agent:g}s budget"
-                    )
-                )
+                self.timeout("agent")
             else:
                 self.fail(e)
             return False
@@ -479,8 +490,11 @@ class Rollout:
             if not self._failed and self._opened:
                 assert runtime is not None
                 trace.timing.finalize.start = time.time()
-                async with boundary(TaskError, "task finalize"):
-                    async with asyncio.timeout(self._timeouts.finalize):
+                try:
+                    async with (
+                        asyncio.timeout(self._timeouts.finalize),
+                        boundary(TaskError, "task finalize"),
+                    ):
                         await invoke(
                             self.task.finalize, {"trace": trace, "runtime": runtime}
                         )
@@ -490,19 +504,24 @@ class Rollout:
                                 self.task.data.artifacts,
                                 max_bytes=self.task.data.artifact_max_bytes,
                             )
+                except TimeoutError:
+                    self.timeout("finalize")
                 now = time.time()
                 trace.timing.finalize.end = now
                 trace.timing.scoring.start = now
                 trace.notify()
-                async with boundary(TaskError, "scoring"):
-                    # Cross-trace judgement runs later, after the runtime is gone.
-                    await asyncio.wait_for(
-                        asyncio.gather(
+                try:
+                    async with (
+                        asyncio.timeout(self._timeouts.scoring),
+                        boundary(TaskError, "scoring"),
+                    ):
+                        # Cross-trace judgement runs later, after the runtime is gone.
+                        await asyncio.gather(
                             self.task.score(trace, runtime),
                             self.harness.score(trace, runtime),
-                        ),
-                        self._timeouts.scoring,
-                    )
+                        )
+                except TimeoutError:
+                    self.timeout("scoring")
                 trace.timing.scoring.end = time.time()
                 trace.notify()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/serve/delta.py

@@ -64,6 +64,7 @@ SCALAR_FIELDS = (
     "is_completed",
     "ok",
     "stop_condition",
+    "is_timeout",
     "timing",
     "num_input_tokens",
     "num_output_tokens",

{verifiers-0.3.2.dev142 → verifiers-0.3.2.dev144}/verifiers/v1/trace.py

@@ -444,6 +444,8 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     """Whether the trace completed successfully."""
     stop_condition: str | None = None
     """What stopped the trace."""
+    is_timeout: bool = False
+    """Whether a stage deadline (setup, agent, finalize, or scoring) expired."""
     errors: list[Error] = Field(default_factory=list)
     """Every error captured across attempts, oldest to newest."""
     timing: Timing = Field(default_factory=Timing)
@@ -755,10 +757,11 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         if response.usage is not None:
             self.extra_usage.append(response.usage)
 
-    def stop(self, condition: str) -> None:
-        """Stop the trace, optionally with a stop condition."""
+    def stop(self, condition: str, override: bool = False) -> None:
+        """Stop the trace with a stop condition. The first condition wins unless
+        `override` replaces it."""
         self.is_completed = True
-        if self.stop_condition is None:
+        if override or self.stop_condition is None:
             self.stop_condition = condition
 
     def split_agent_time(self) -> None:
@@ -770,6 +773,13 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         span.model.duration = min(model, span.duration)
         span.harness.duration = span.duration - span.model.duration
 
+    def record_timeout(self, stage: str) -> None:
+        """Record a stage deadline's expiry, and stop the trace as `<stage>_timeout`.
+        The deadline is what ended the trace, so it replaces any earlier stop
+        condition (a finalize deadline can expire after `agent_completed`)."""
+        self.is_timeout = True
+        self.stop(f"{stage}_timeout", override=True)
+
     def record_error(self, error: Exception) -> None:
         """Record an error, and stop the trace as failed."""
         self.errors.append(