verifiers 0.2.2.dev24__tar.gz → 0.2.2.dev25__tar.gz

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev24
+Version: 0.2.2.dev25
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/tests/v1/conftest.py

@@ -37,7 +37,7 @@ from pathlib import Path
 import pytest
 
 import verifiers.v1 as vf
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.loaders import load_environment
 from verifiers.v1.cli.eval.runner import run_eval
 from verifiers.v1.trace import Trace
@@ -123,16 +123,16 @@ def _eval_config(
     max_turns: int | None = 4,
     rollout_timeout: float = 180,
     taskset_overrides: dict | None = None,
-    harness_overrides: dict | None = None,
+    runtime: dict | None = None,
     env: dict | None = None,
     pool: dict | None = None,
     model: str | None = None,
     reasoning_effort: str | None = None,
 ) -> EvalConfig:
     """Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
-    (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` / `harness_overrides`
-    are merged onto the `{id: ...}` config (placement, runtime, etc.); `model` overrides the default
-    text model (e.g. a VLM for an image task).
+    (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the
+    `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
+    harness one); `model` overrides the default text model (e.g. a VLM for an image task).
 
     `temperature=0` (greedy) makes the run reproducible; `max_tokens` is generous headroom,
     not a target — these trivial tasks finish in a few hundred tokens, so capping tighter only
@@ -145,9 +145,11 @@ def _eval_config(
     env_cfg = dict(env or {})
     _configure_prime_runtimes(taskset_cfg)
     if harness:
-        harness_cfg = {"id": harness, **(harness_overrides or {})}
-        _configure_prime_runtimes(harness_cfg)
-        env_cfg.setdefault("agent", {})["harness"] = harness_cfg
+        env_cfg.setdefault("agent", {})["harness"] = {"id": harness}
+    if runtime:
+        runtime_cfg = dict(runtime)
+        _configure_prime_runtimes(runtime_cfg)
+        env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
     # Per-run caps live on the seats: resolve the env's declared roles and cap
     # each one (a test's own seat dict wins over the shared defaults).
     config_cls = vf.env_config_type(taskset, env_cfg.get("id", ""))

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/tests/v1/test_configs.py

@@ -11,7 +11,7 @@ from pathlib import Path
 
 import pytest
 
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 
 CONFIGS = sorted(
     p

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/tests/v1/test_e2e.py

@@ -100,7 +100,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-v1",
         harness=harness,
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=2,
     )
@@ -109,7 +109,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
     # The seat's resolved identity rides the trace (policy metadata for trainers).
-    assert trace.agent is not None and trace.agent.sampling.temperature == 0
+    assert trace.agent is not None and trace.agent.config.sampling.temperature == 0
     # Every sampled turn has one per-call record, linked to its assistant node.
     sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
     assert [c.node for c in trace.calls if c.error is None] == sampled
@@ -130,7 +130,7 @@ async def test_user(run_v1, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-user-sim-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=6,
     )
@@ -189,7 +189,7 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-acp-resume-v1",
         harness=harness,
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=8,
         max_tokens=8192,
@@ -219,7 +219,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-tool-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=6,
         taskset_overrides={"task": {"tools": tool_runtime}},
@@ -246,7 +246,7 @@ async def test_tool_state(run_v1, harness_runtime, tool_runtime, tmp_path):
     (trace,) = await run_v1(
         "counter-tool-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=8,
         taskset_overrides={"task": {"tools": tool_runtime}},
@@ -267,7 +267,7 @@ async def test_shared_tool_isolation(
     traces = await run_v1_server(
         "scratchpad-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         num_tasks=2,
         n=1,
@@ -287,7 +287,7 @@ async def test_tool_response_image(run_v1, tmp_path):
     (trace,) = await run_v1(
         "tool-response-image-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         model="openai/gpt-5.6-luna",
         reasoning_effort="none",
         output_dir=tmp_path,
@@ -314,7 +314,7 @@ async def test_rubric_judge(run_v1, tmp_path):
     (trace,) = await run_v1(
         "echo-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=tmp_path,
         taskset_overrides={"task": {"judges": [{"id": "rubric", "path": str(rubric)}]}},
         max_turns=2,
@@ -332,7 +332,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-agentic-v1",
         harness=harness,
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=10,
     )
@@ -406,7 +406,7 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
             # The judge reads the transcript and reasons before it writes the
             # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
-                "harness": {"runtime": {"type": "docker"}},
+                "runtime": {"type": "docker"},
                 "max_output_tokens": 8192,
             },
         },
@@ -551,13 +551,13 @@ async def test_replay_round_trip(run_v1, tmp_path):
 
     from verifiers.v1.cli.output import CONFIG_FILE
     from verifiers.v1.cli.replay import run_replay
-    from verifiers.v1.configs.replay import ReplayConfig
+    from verifiers.v1.configs.cli.replay import ReplayConfig
 
     run_dir = tmp_path / "run"
     (source,) = await run_v1(
         "reverse-text-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=run_dir,
         max_turns=2,
     )

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/tests/v1/test_legacy.py

@@ -22,7 +22,7 @@ async def test_v0_single_turn_matches_v1_shape(run_v0, run_v1, tmp_path):
     (v0,) = await run_v0("echo-v0", output_dir=tmp_path / "v0")
     (v1,) = await run_v1(
         "echo-v1",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=tmp_path / "v1",
         max_turns=2,
     )
@@ -36,7 +36,7 @@ async def test_v0_multi_turn_matches_v1_shape(run_v0, run_v1, tmp_path):
     (v0,) = await run_v0("echo-multi-v0", output_dir=tmp_path / "v0")
     (v1,) = await run_v1(
         "echo-v1",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=tmp_path / "v1",
         max_turns=2,
     )

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/__init__.py

@@ -13,25 +13,16 @@ from verifiers.v1.clients import (
     resolve_client,
 )
 from verifiers.v1.decorators import metric, reward, stop, tool
-from verifiers.v1.agent import (
-    Agent,
-    AgentConfig,
-    Agents,
-    Interaction,
-    Segment,
-    make_agent,
-)
-from verifiers.v1.configs.env import (
+from verifiers.v1.configs.agent import AgentConfig
+from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
+from verifiers.v1.configs.cli.env import (
     ElasticPoolConfig,
     EnvServerConfig,
     StaticPoolConfig,
     pool_serve_kwargs,
 )
-from verifiers.v1.env import (
-    EnvConfig,
-    Env,
-    default_agent_harness,
-)
+from verifiers.v1.configs.env import EnvConfig, default_agent_harness
+from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
 from verifiers.v1.errors import (
     EnvError,
@@ -44,15 +35,10 @@ from verifiers.v1.errors import (
     ToolsetError,
     TunnelError,
 )
-from verifiers.v1.harness import Harness, HarnessConfig
-from verifiers.v1.judge import (
-    Judge,
-    JudgeConfig,
-    JudgeResponse,
-    Judges,
-    JudgeSamplingConfig,
-    JudgeView,
-)
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.configs.judge import JudgeConfig, JudgeSamplingConfig, Judges
+from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
 from verifiers.v1.judges import (
     ReferenceJudge,
     ReferenceJudgeConfig,
@@ -86,7 +72,7 @@ from verifiers.v1.scoring import (
     read_answer_file_or_last_reply as read_answer_file_or_last_reply,
     verify_boxed_math_answer as verify_boxed_math_answer,
 )
-from verifiers.v1.retries import RetryConfig
+from verifiers.v1.configs.retries import RetryConfig
 from verifiers.v1.utils.git import (
     PATCH_CAP_BYTES as PATCH_CAP_BYTES,
     capture_patch as capture_patch,
@@ -102,15 +88,10 @@ from verifiers.v1.runtimes import (
     SubprocessConfig,
 )
 from verifiers.v1.state import State, StateT
-from verifiers.v1.task import (
-    Task,
-    TaskConfig,
-    TaskData,
-    TaskResources,
-    TaskTimeout,
-    WireTaskData,
-)
-from verifiers.v1.taskset import Taskset, TasksetConfig
+from verifiers.v1.configs.task import TaskConfig
+from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData
+from verifiers.v1.configs.taskset import TasksetConfig
+from verifiers.v1.taskset import Taskset
 from verifiers.v1.mcp import (
     Toolset,
     SharedToolsetConfig,

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/agent.py

@@ -14,20 +14,18 @@ from contextlib import asynccontextmanager, nullcontext
 from dataclasses import dataclass
 from typing import AsyncIterator
 
-from pydantic import SerializeAsAny, model_validator
-from pydantic_config import BaseConfig
 
+from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
 from verifiers.v1.clients import (
     Client,
-    ClientConfig,
     EvalClientConfig,
     ModelContext,
     resolve_client,
 )
-from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
-from verifiers.v1.retries import RetryConfig, backoff, trace_should_retry
+from verifiers.v1.retries import backoff, trace_should_retry
 from verifiers.v1.rollout import RolloutRun, _as_messages
 from verifiers.v1.runtimes import (
     NetworkPolicyConfig,
@@ -44,7 +42,6 @@ from verifiers.v1.types import (
     AssistantMessage,
     Messages,
     Sampling,
-    SamplingConfig,
     ToolMessage,
     UserMessage,
 )
@@ -54,57 +51,9 @@ from verifiers.v1.utils.compile import (
     validate_pairing,
 )
 
-logger = logging.getLogger(__name__)
-
+__all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 
-class TimeoutConfig(BaseConfig):
-    """Per-agent wall-clock timeouts per rollout stage, in seconds (None = no
-    limit); each stage falls back to the task's own `TaskTimeout` when unset. An
-    interaction's rollout budget is cumulative across its active harness segments
-    and pauses while the caller computes the next user turn."""
-
-    setup: float | None = None  # one shared budget: task setup + provisioning
-    rollout: float | None = None
-    finalize: float | None = None
-    scoring: float | None = None
-
-
-class AgentConfig(BaseConfig):
-    """One env agent: who plays it, and its per-run caps. It pins only what
-    makes it a different actor; everything unpinned falls back — the model context
-    to the run's own, the harness to the taskset's default."""
-
-    harness: SerializeAsAny[HarnessConfig] | None = None
-    """The agent's program + runtime policy (None = the taskset's default harness)."""
-    model: str | None = None
-    """Model id (None = the run's model, i.e. the policy under evaluation/training)."""
-    client: ClientConfig | None = None
-    """Endpoint override (None = the run's client)."""
-    sampling: SamplingConfig | None = None
-    """Sampling override (None = the run's sampling)."""
-    timeout: TimeoutConfig = TimeoutConfig()
-    retries: RetryConfig = RetryConfig()
-    """Whole-run retries: rerun this agent's rollout while its trace ends with a
-    retryable error (never into a borrowed box)."""
-    max_turns: int | None = None
-    """Max model turns per run (None = no limit). Framework-enforced (the
-    interception server refuses turns past it), so it applies to any harness."""
-    max_input_tokens: int | None = None
-    max_output_tokens: int | None = None
-    max_total_tokens: int | None = None
-    """Token caps per run (None = no limit); framework-enforced between turns."""
-
-    @model_validator(mode="before")
-    @classmethod
-    def _resolve_harness(cls, data):
-        """Narrow a pinned `harness` to its concrete config type by `id` (absent
-        stays None = the taskset's default). The lazy import keeps class-body
-        `AgentConfig()` defaults constructible while this module initializes."""
-        if isinstance(data, dict) and data.get("harness") is not None:
-            from verifiers.v1.loaders import harness_config_type, narrow_plugin_field
-
-            narrow_plugin_field(data, "harness", harness_config_type, "bash")
-        return data
+logger = logging.getLogger(__name__)
 
 
 def _check_borrowed_placement(
@@ -283,8 +232,8 @@ class Agent:
 
     Built from an `AgentConfig` alone; `client=`/`interception=` inject live
     resources to borrow — agents on one endpoint should share one `Client`, and a
-    live `Interception`'s owner keeps its lifecycle. The harness config's
-    `runtime` is a *policy*: each `run` provisions a fresh box from it, resolved
+    live `Interception`'s owner keeps its lifecycle. The config's `runtime` is a
+    *policy*: each `run` provisions a fresh box from it, resolved
     per task; `run(runtime=...)` places the run into an existing box instead
     (borrowed boxes are never started or torn down by the run)."""
 
@@ -302,21 +251,26 @@ class Agent:
                 "AgentConfig.model is unset; an Agent needs a pinned model "
                 "(inside an env the run's own model fills it in)"
             )
-        harness_config = config.harness
-        if harness_config is None:
-            harness_config = harness_config_type("bash")(id="bash")
+        # Resolve the unpinned identity fields into the config: it is the agent's
+        # full identity — what stamps onto every trace (`AgentInfo.config`).
+        if config.harness is None:
+            config = config.model_copy(
+                update={"harness": harness_config_type("bash")(id="bash")}
+            )
+        if config.sampling is None:
+            config = config.model_copy(update={"sampling": Sampling()})
         self.config = config
-        self.harness = load_harness(harness_config)
+        self.harness = load_harness(config.harness)
         self._owns_client = client is None
         if self._owns_client:
             client = resolve_client(config.client or EvalClientConfig())
         self.ctx = ModelContext(
             model=config.model,
             client=client,
-            sampling=config.sampling if config.sampling is not None else Sampling(),
+            sampling=config.sampling,
         )
         self._closed = False
-        self.runtime_config: RuntimeConfig = self.harness.config.runtime
+        self.runtime_config: RuntimeConfig = config.runtime
         self.interception = interception
         self.limits = RolloutLimits(
             max_turns=config.max_turns,
@@ -575,6 +529,7 @@ class Agent:
             else task.data.timeout.harness
         )
         return dict(
+            agent_config=self.config,
             harness=self.harness,
             ctx=self.ctx,
             runtime_config=runtime_config,

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/dashboard/eval.py

@@ -16,7 +16,7 @@ from verifiers.v1.cli.dashboard.base import live_view
 from verifiers.v1.cli.output import output_path
 from verifiers.v1.utils.install import env_name
 from verifiers.v1.utils.interrupt import cleaning_up
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import RunSlot
 from verifiers.v1.trace import Trace
 from verifiers.v1.types import Usage
@@ -73,7 +73,7 @@ _MARK = {
 def _seat_value(config: EvalConfig, read):
     """A cap as the overview shows it: the declared seats' shared value, the
     string 'per-seat' when they disagree (caps live on the seats)."""
-    from verifiers.v1.env import _declared_agent_configs
+    from verifiers.v1.configs.env import _declared_agent_configs
 
     values = {read(spec) for spec in _declared_agent_configs(config.env).values()}
     return values.pop() if len(values) == 1 else "per-seat"
@@ -163,8 +163,9 @@ def _warning(config: EvalConfig) -> Text | None:
     from verifiers.v1.loaders import harness_class
 
     if any(
-        h.runtime.type == "subprocess" and harness_class(h.id).EXECUTES_CODE
-        for h in config.env.agent_harnesses().values()
+        getattr(config.env, role).runtime.type == "subprocess"
+        and harness_class(h.id).EXECUTES_CODE
+        for role, h in config.env.agent_harnesses().items()
     ):
         return Text(
             "warning  Runs on the local system; local files and settings may affect this "
@@ -185,7 +186,7 @@ def overrides(
     `model_validate(config.toml)`, and that toml is dumped with `exclude_none` (every field), so
     `model_fields_set` would flag them all. `default` is the reference instance, threaded through
     recursion so a pinned nested default (`taskset.task.tools`) reads as
-    unchanged. `skip` holds dotted paths (`harness.runtime.type`)."""
+    unchanged. `skip` holds dotted paths (`runtime.type`)."""
     segments: list[str] = []
     fields = type(config).model_fields
     for field in sorted(fields):
@@ -237,9 +238,11 @@ def Overview(config: EvalConfig) -> Table:
         env_label = f"{env_name(config.env.id)}+{env_label}"
     # One seat story when every seat resolves the same way (the common case); one
     # row per seat when they diverge (a judge on its own harness/runtime).
+    runtimes = {role: getattr(config.env, role).runtime.type for role in seats}
     stories = list(
         dict.fromkeys(
-            f"{h.name} harness  ·  {h.runtime.type} runtime" for h in seats.values()
+            f"{h.name} harness  ·  {runtimes[role]} runtime"
+            for role, h in seats.items()
         )
     )
     if len(stories) == 1:
@@ -247,19 +250,23 @@ def Overview(config: EvalConfig) -> Table:
     else:
         grid.add_row("env", env_label)
         for role, h in seats.items():
-            grid.add_row(f"  {role}", f"{h.name} harness  ·  {h.runtime.type} runtime")
+            grid.add_row(f"  {role}", f"{h.name} harness  ·  {runtimes[role]} runtime")
     model = f"{config.model}  ({sampling})" if sampling else config.model
     grid.add_row("model", f"{model}  via {config.client.base_url}")
     # Non-default knobs the user set, one row each when non-empty. `escape` the cell: an override
     # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be
-    # parsed as styling and dropped. `id` is in the `env` row; harness `runtime.type` too (hidden
-    # here), but only for the harness — `taskset.task.tools.runtime.type` has no other display.
+    # parsed as styling and dropped. `id` is in the `env` row; the seat's `runtime.type` too
+    # (hidden here), but only for the seat — `taskset.task.tools.runtime.type` has no other display.
     if taskset_over := overrides(taskset, skip=frozenset({"id"})):
         grid.add_row("taskset", escape("  ·  ".join(taskset_over)))
     for role, h in seats.items():
-        if harness_over := overrides(h, skip=frozenset({"id", "runtime.type"})):
+        if harness_over := overrides(h, skip=frozenset({"id"})):
             label = f"{role}.harness" if len(seats) > 1 else "harness"
             grid.add_row(label, escape("  ·  ".join(harness_over)))
+        runtime = getattr(config.env, role).runtime
+        if runtime_over := overrides(runtime, skip=frozenset({"type"})):
+            label = f"{role}.runtime" if len(seats) > 1 else "runtime"
+            grid.add_row(label, escape("  ·  ".join(runtime_over)))
     limits, timeouts = _aligned([_limits(config), _timeouts(config)])
     grid.add_row("limits", limits)
     grid.add_row("timeouts", timeouts)
@@ -753,7 +760,8 @@ def _render(
             now,
             "/".join(
                 dict.fromkeys(
-                    h.runtime.type for h in config.env.agent_harnesses().values()
+                    getattr(config.env, role).runtime.type
+                    for role in config.env.agent_harnesses()
                 )
             )
             or "subprocess",

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/dashboard/validate.py

@@ -13,7 +13,7 @@ from rich.table import Table
 from rich.text import Text
 
 from verifiers.v1.cli.dashboard.base import live_view
-from verifiers.v1.configs.validate import ValidateConfig
+from verifiers.v1.configs.cli.validate import ValidateConfig
 from verifiers.v1.utils.format import format_time
 
 _STYLE = {

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/debug.py

@@ -22,13 +22,13 @@ from verifiers.v1.cli.resolve import (
     references_config_file,
     with_positional_taskset,
 )
-from verifiers.v1.configs.debug import DebugConfig
+from verifiers.v1.configs.cli.debug import DebugConfig
 from verifiers.v1.decorators import invoke
 from verifiers.v1.utils.compile import resolve_runtime_config
 from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
-from verifiers.v1.trace import Error, Trace, TraceTask
+from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 
@@ -202,6 +202,14 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     trace = Trace(
         task=TraceTask(type=type(task).__name__, data=task.data),
         state=state_cls(type(task))(),
+        # No agent plays here (no model, no harness): the seat records the
+        # runtime policy, and the provisioned box lands on it below (id, resolved
+        # type, cache status).
+        agent=AgentInfo(
+            config=vf.AgentConfig(runtime=config.runtime),
+            name="debug",
+            trainable=False,
+        ),
     )
     debug = {
         "task": task_info(task),
@@ -214,7 +222,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         resolve_runtime_config(config.runtime, task),
         name=f"debug-{task.data.idx}-{uuid4().hex[:8]}",
     )
-    trace.runtime = runtime.info
+    trace.agent.runtime = runtime.info
     setup_timeout = (
         config.timeout.setup
         if config.timeout.setup is not None

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/eval/main.py

@@ -19,7 +19,7 @@ from verifiers.v1.cli.resolve import (
 )
 from verifiers.v1.cli.eval.resume import load_resume_config, split_resume
 from verifiers.v1.cli.eval.runner import run_eval
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 
 logger = logging.getLogger(__name__)
 

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/eval/resume.py

@@ -22,7 +22,7 @@ from typing import TypeVar
 from pydantic_core import from_json
 
 from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.trace import WireTrace
 

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/eval/runner.py

@@ -6,7 +6,7 @@ import logging
 import time
 
 from verifiers.v1.clients import ModelContext, resolve_client
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.cli.eval import resume
 from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.output import (
@@ -108,7 +108,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
     from functools import partial
 
     from verifiers.v1.utils.logging import setup_logging
-    from verifiers.v1.configs.env import pool_serve_kwargs
+    from verifiers.v1.configs.cli.env import pool_serve_kwargs
     from verifiers.v1.serve import EnvClient, env_config_data, serve_env
 
     legacy = config.is_legacy

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/init.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from pydantic_config import cli
 
-from verifiers.v1.configs.init import InitConfig
+from verifiers.v1.configs.cli.init import InitConfig
 
 USAGE = (
     "usage: uv run init <name> [--path ./environments] [-T/--add-tool] "

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/output.py

@@ -17,7 +17,7 @@ from pathlib import Path
 import tomli_w
 from pydantic import BaseModel, TypeAdapter
 
-from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.trace import Trace
 from verifiers.v1.utils.aio import run_shielded

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/replay.py

@@ -27,9 +27,10 @@ from verifiers.v1.cli.output import (
     save_config,
     write_config,
 )
-from verifiers.v1.configs.replay import ReplayConfig
+from verifiers.v1.configs.cli.replay import ReplayConfig
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task, WireTaskData, task_data_cls
+from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.trace import Trace
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
@@ -86,7 +87,9 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
     # An episode may hold no traces (its env hooks failed before any agent ran);
     # there's nothing to re-score, so it drops out in the flatten. Each kept trace
     # remembers its episode's env identity — the replay output re-stamps it.
-    episodes = read_episodes(source, Trace[WireTaskData, state_cls(task_cls)])
+    episodes = read_episodes(
+        source, Trace[WireTaskData, state_cls(task_cls), WireAgentConfig]
+    )
     sourced = [(trace, e.env) for e in episodes for trace in e.traces]
     if config.num_traces is not None:
         sourced = sourced[: config.num_traces]

{verifiers-0.2.2.dev24 → verifiers-0.2.2.dev25}/verifiers/v1/cli/serve.py

@@ -13,8 +13,8 @@ from verifiers.v1.cli.resolve import (
     references_config_file,
     with_positional_taskset,
 )
-from verifiers.v1.configs.serve import ServeConfig
-from verifiers.v1.configs.env import pool_serve_kwargs
+from verifiers.v1.configs.cli.serve import ServeConfig
+from verifiers.v1.configs.cli.env import pool_serve_kwargs
 from verifiers.v1.serve import serve_env
 
 USAGE = "usage: uv run serve [<taskset-id>] [--env.id <id>] [--id <env-id> (legacy)] [options] [@ file.toml]"