verifiers 0.4.1.dev14__tar.gz → 0.4.1.dev19__tar.gz

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev14
+Version: 0.4.1.dev19
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/tests/integration/grid.py

@@ -12,7 +12,7 @@ capability the code declares, never from a list of names, so the grid follows th
 - `Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support
   (`SUPPORTS_RESUME` or a native session): the behavior needs a shell, MCP tools,
   skills, or a resumable exchange
-- `Runtime.supports_user`, `Runtime.supports_expose`, `NetworkPolicyConfig`: the
+- `RuntimeBackend.supports_user`, `RuntimeBackend.supports_expose`, `NetworkPolicyConfig`: the
   behavior needs a user switch, a tool server reachable from another runtime, or a
   network policy
 - `podman` / `apptainer` need their engine on the host
@@ -39,7 +39,7 @@ from pydantic import TypeAdapter
 import verifiers.v1 as vf
 from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.harness import Harness
-from verifiers.v1.runtimes import RuntimeConfig, _runtime_cls
+from verifiers.v1.runtimes import RuntimeConfig, _backend_cls
 from verifiers.v1.utils.loaders import harness_class
 
 HARNESSES: list[str | dict] = [
@@ -92,7 +92,7 @@ def runtime_config(runtime: str, harness: str | dict | None = None) -> dict:
 
 
 def runtime_cls(runtime: str) -> type:
-    return _runtime_cls(TypeAdapter(RuntimeConfig).validate_python({"type": runtime}))
+    return _backend_cls(TypeAdapter(RuntimeConfig).validate_python({"type": runtime}))
 
 
 def runtime_supports_user(runtime: str) -> bool:

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/tests/integration/test_agentic_judge.py

@@ -1,9 +1,7 @@
-"""The agentic judge envs verify a solver empirically in a sandbox: `shared-agentic-judge`
-reuses the solver's restricted box, `agentic-judge` restores the solver's artifact in a
-fresh box."""
+"""Agentic judging uses shared or isolated scoring to verify a solver in a sandbox."""
 
 
-async def test_env_id_shared_agentic_judge(run, tmp_path):
+async def test_agentic_judge_shared_verifier(run, tmp_path):
     """The shared agentic judge provisions one restricted solver-owned box and
     safely reuses it for the judge's empirical verification."""
     policy = tmp_path / "judge_policy.txt"
@@ -12,10 +10,11 @@ async def test_env_id_shared_agentic_judge(run, tmp_path):
         "echo-v1",
         harness=None,
         env={
-            "id": "shared-agentic-judge",
+            "id": "agentic-judge",
             "solver": {
                 "harness": {"id": "bash"},
                 "runtime": {"type": "docker", "block": ["example.com"]},
+                "scoring": {"mode": "shared"},
             },
             "judge": {
                 "harness": {"id": "bash"},
@@ -60,6 +59,7 @@ async def test_env_id_agentic_judge(run, tmp_path):
             "solver": {
                 "harness": {"id": "bash"},
                 "runtime": {"type": "docker", "block": ["example.com"]},
+                "scoring": {"mode": "isolated"},
             },
             "judge": {
                 "harness": {"id": "bash"},

verifiers-0.4.1.dev19/tests/integration/test_compose_scoring.py

@@ -0,0 +1,169 @@
+"""Compose scoring uses live shared services or a released, collected submission."""
+
+import shutil
+
+import pytest
+
+import verifiers.v1 as vf
+from verifiers.v1.configs.agent import TimeoutConfig
+from verifiers.v1.runtimes.container import cli
+from verifiers.v1.tasksets.harbor.taskset import (
+    CollectHook,
+    HarborArtifact,
+    HarborData,
+    HarborTask,
+)
+
+pytestmark = [
+    pytest.mark.docker,
+    pytest.mark.skipif(not shutil.which("docker"), reason="needs Docker"),
+]
+
+IMAGE = "python:3.11-slim"
+COMPOSE = f"""
+services:
+  main:
+    image: {IMAGE}
+    working_dir: /app
+    command: [sleep, infinity]
+    depends_on:
+      sidecar:
+        condition: service_healthy
+  sidecar:
+    environment:
+      COLLECTION_SCOPE: sidecar
+    image: {IMAGE}
+    command: [sh, -c, "mkdir -p /srv /evidence; echo 42 > /srv/answer; cd /srv; exec python -m http.server 8000"]
+    healthcheck:
+      test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8000/answer')"]
+      interval: 1s
+      timeout: 2s
+      retries: 30
+"""
+
+
+class ComposeHarness(vf.Harness):
+    async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls, data):
+        return await runtime.run_program(
+            [
+                "python",
+                "-c",
+                (
+                    "import urllib.request; "
+                    "open('/app/answer', 'wb').write(urllib.request.urlopen('http://sidecar:8000/answer').read())"
+                ),
+            ],
+            {},
+        )
+
+
+class ComposeTask(HarborTask):
+    async def setup(self, trace, runtime, deployment=None):
+        await super().setup(runtime)
+        if deployment is not None:
+            trace.info["solver_containers"] = [
+                runtime.info.id for runtime in deployment.target_backends.values()
+            ]
+        else:
+            # Fresh verifier setup happens only after the solver project is removed.
+            for container in trace.info["solver_containers"]:
+                result = await cli("docker", "inspect", container)
+                assert result.exit_code != 0, "solver service survived into grading"
+            trace.info["solver_released_before_grading"] = True
+
+    async def finalize_services(self, trace, targets):
+        result = await cli(
+            "docker",
+            "inspect",
+            "--format",
+            "{{.State.Running}}",
+            targets["main"].info.id,
+        )
+        assert result.stdout.strip() == "false", (
+            "workspace must stop before sidecar collection"
+        )
+        trace.info["collected_sidecars"] = True
+        await super().finalize_services(trace, targets)
+
+
+@pytest.mark.parametrize(
+    ("mode", "collect_fails", "scoring_timeout"),
+    [
+        ("shared", True, False),
+        ("isolated", False, False),
+        ("isolated", True, False),
+        ("isolated", False, True),
+    ],
+    ids=[
+        "shared-skips-sidecars",
+        "isolated-restores-sidecars",
+        "isolated-collect-failure",
+        "isolated-scoring-timeout",
+    ],
+)
+async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
+    environment = tmp_path / "environment"
+    environment.mkdir()
+    (environment / "docker-compose.yaml").write_text(COMPOSE)
+    tests = tmp_path / "tests"
+    tests.mkdir()
+    (tests / "test.sh").write_text(
+        "set -eu\n"
+        'test "$(cat /app/answer)" = 42\n'
+        'if [ "$EXPECT_SIDECAR" = 1 ]; then test "$(cat /evidence/result)" = 42; fi\n'
+        + ("sleep 30\n" if scoring_timeout else "")
+        + "echo 1 > /logs/verifier/reward.txt\n"
+    )
+    task = ComposeTask(
+        HarborData(
+            prompt="Copy the sidecar answer",
+            task_dir=str(tmp_path),
+            env={"COLLECTION_SCOPE": "workspace"},
+            # Exercise agent overrides in both directions, not just task defaults.
+            scoring=vf.ScoringConfig(mode="isolated" if mode == "shared" else "shared"),
+            artifacts=[
+                HarborArtifact(source="/app/answer"),
+                HarborArtifact(source="/evidence/result", service="sidecar"),
+            ],
+            collect=[
+                CollectHook(
+                    service="sidecar",
+                    command="exit 17"
+                    if collect_fails
+                    else 'test "$COLLECTION_SCOPE" = sidecar && cp /srv/answer /evidence/result',
+                )
+            ],
+        )
+    )
+    config = vf.AgentConfig(
+        model="unused",
+        runtime=vf.DockerConfig(),
+        deployment=vf.DeploymentConfig(trust_compose=True),
+        scoring=vf.ScoringConfig(
+            mode=mode,
+            env={"EXPECT_SIDECAR": "1" if mode == "isolated" else "0"},
+            retries=vf.RetryConfig(max_retries=0),
+        ),
+        timeout=TimeoutConfig(
+            setup=180, finalize=30, scoring=1 if scoring_timeout else 30
+        ),
+    )
+    async with vf.Agent(config) as agent:
+        agent.harness = ComposeHarness(vf.HarnessConfig())
+        trace = await agent.run(task)
+    for container in trace.info.get("solver_containers", []):
+        assert (await cli("docker", "inspect", container)).exit_code != 0
+    if mode == "isolated" and collect_fails:
+        assert not trace.ok
+        assert "collect hook failed" in str(trace.errors)
+        assert "solver_released_before_grading" not in trace.info
+    elif scoring_timeout:
+        assert trace.ok and trace.is_timeout and not trace.errors
+        assert trace.stop_condition == "scoring_timeout"
+        assert all(reward is None for reward in trace.rewards.values())
+    else:
+        assert trace.ok and trace.reward == 1, trace.errors
+        assert bool(trace.info.get("collected_sidecars")) == (mode == "isolated")
+        assert bool(trace.info.get("solver_released_before_grading")) == (
+            mode == "isolated"
+        )

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/tests/unit/test_agent.py

@@ -53,6 +53,7 @@ async def test_failed_segment_does_not_reuse_prior_root_reply():
     run._agent_time_remaining = None
     run._timeouts = RolloutTimeouts()
     run._harness_session = FailingSession()
+    run.deployment = SimpleNamespace(run=lambda turn: turn)
     run._session = SimpleNamespace(
         request_interceptors=[],
         error=None,

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/tests/unit/utils/test_loaders.py

@@ -40,9 +40,7 @@ def test_unknown_harness_names_the_builtins() -> None:
         harness_class("no-such-harness")
 
 
-@pytest.mark.parametrize(
-    "env_id", ["best-of-n", "user-sim", "agentic-judge", "shared-agentic-judge"]
-)
+@pytest.mark.parametrize("env_id", ["best-of-n", "user-sim", "agentic-judge"])
 def test_bundled_envs_load_with_a_role_typed_config(env_id: str) -> None:
     cls = environment_class("", env_id)
     assert issubclass(cls, vf.Env)

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/verifiers/v1/__init__.py

@@ -61,6 +61,7 @@ from verifiers.v1.errors import (
     ToolsetError,
     TunnelError,
 )
+from verifiers.v1.grading import grade_submission
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
@@ -371,6 +372,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "PATCH_CAP_BYTES",
     "capture_patch",
     "resolve_head",
+    "grade_submission",
     # grading artifacts
     "ARTIFACTS_DIR",
     "Artifact",
@@ -396,3 +398,19 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
 # silent until an app opts in: a NullHandler on the package root absorbs records
 # so nothing is emitted (and no "no handler" warning) unless handlers are added.
 _logging.getLogger(__name__).addHandler(_logging.NullHandler())
+
+from verifiers.v1.runtimes.compose import ComposeSpec
+from verifiers.v1.runtimes.deployment import (
+    Deployment,
+    DeploymentConfig,
+)
+
+__all__ += [
+    "ComposeSpec",
+    "Deployment",
+    "DeploymentConfig",
+]
+
+from verifiers.v1.configs.scoring import ScoringConfig
+
+__all__ += ["ScoringConfig"]

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/verifiers/v1/agent.py

@@ -9,8 +9,9 @@ server; un-entered, each run brings its own."""
 
 import asyncio
 import logging
-from collections.abc import AsyncIterator, Callable, Mapping
+from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
 from contextlib import asynccontextmanager, nullcontext
+from contextvars import ContextVar
 from dataclasses import dataclass, field, replace
 from typing import Generic, Literal, Self, cast
 from weakref import WeakValueDictionary
@@ -37,7 +38,7 @@ from verifiers.v1.runtimes import (
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
-    provision_runtime,
+    provision_backend,
     runtime_is_local,
 )
 from verifiers.v1.session import RolloutLimits
@@ -61,6 +62,11 @@ __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 
 logger = logging.getLogger(__name__)
 
+# Only nested runs in this scoring attempt participate in its rollback.
+_scoring_runs: ContextVar[list[tuple["_EpisodeAgent", Trace]] | None] = ContextVar(
+    "scoring_runs", default=None
+)
+
 
 def resolve_rollout_timeouts(timeout: TimeoutConfig, task: Task) -> RolloutTimeouts:
     """Apply an agent's stage-timeout precedence to one task."""
@@ -657,6 +663,7 @@ class Agent:
         tools: Mapping[str, SharedToolServer] | None = None,
         on_trace: Callable[[Trace], None] | None = None,
         collect_artifacts: bool = False,
+        on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None,
     ) -> Trace:
         """Run this agent on `task` once and return the trace: one segment — the
         program runs on the task's prompt until it exits (a multi-turn exchange
@@ -667,7 +674,8 @@ class Agent:
         declared artifacts after its finalizer while its container runtime is still
         alive. Retries whole while the trace ends with a retryable error
         (`config.retries`) — never into a borrowed box; the final trace keeps earlier
-        attempts' errors."""
+        attempts' errors. `on_score(trace, runtime)` runs after task scoring in
+        the selected verifier workspace, inside its timeout and retry boundary."""
         if self._closed:
             raise RuntimeError("Agent is closed; create a new agent")
         retry = self.config.retries
@@ -675,7 +683,7 @@ class Agent:
         history: list = []
         for attempt in range(retry_state.max_retries + 1):
             trace = await self._run_once(
-                task, runtime, tools, on_trace, collect_artifacts
+                task, runtime, tools, on_trace, collect_artifacts, on_score
             )
             if attempt == retry_state.max_retries or trace.ok:
                 break
@@ -711,6 +719,7 @@ class Agent:
         shared_tools: Mapping[str, SharedToolServer] | None,
         on_trace: Callable[[Trace], None] | None,
         collect_artifacts: bool,
+        on_score: Callable[[Trace, Runtime], Awaitable[None]] | None,
     ) -> Trace:
         params = self._rollout_params(task, runtime, dict(shared_tools or {}))
         if collect_artifacts and isinstance(params["runtime_config"], SubprocessConfig):
@@ -722,6 +731,7 @@ class Agent:
             task=task,
             on_trace=on_trace,
             collect_artifacts=collect_artifacts,
+            on_score=on_score,
             **params,
         )
         try:
@@ -850,16 +860,16 @@ class Agent:
     @asynccontextmanager
     async def provision(self, task: Task | None = None) -> AsyncIterator[Runtime]:
         """Provision (and on exit tear down) a box from this agent's runtime
-        policy, resolved for `task` when given; share it via `run(..., runtime=box)`."""
+        policy, resolved for `task` when given; share it via `run(..., runtime=runtime)`."""
         config = (
             resolve_runtime_config(self.runtime_config, task, self._warned_resources)
             if task is not None
             else self.runtime_config
         )
-        async with provision_runtime(config) as runtime:
+        async with provision_backend(config) as backend:
             # Keep sandbox startup task-neutral: this box may later host another task.
-            runtime.env = dict(task.runtime_env()) if task is not None else {}
-            yield runtime
+            backend.env = dict(task.runtime_env()) if task is not None else {}
+            yield backend.bind()
 
 
 class _EpisodeAgent(Agent):
@@ -914,6 +924,8 @@ class _EpisodeAgent(Agent):
             if last is not None and self._on_discard is not None:
                 self._on_discard(last)
             last = trace
+            if (runs := _scoring_runs.get()) is not None:
+                runs.append((self, trace))
             if self._on_trace is not None:
                 self._on_trace(trace)
             if on_trace is not None:
@@ -921,6 +933,42 @@ class _EpisodeAgent(Agent):
 
         return watch
 
+    def _scoring_callback(self, callback):
+        if callback is None:
+            return None
+
+        async def score(trace, runtime):
+            parent_runs = _scoring_runs.get()
+            runs = []
+            token = _scoring_runs.set(runs)
+            # The solver is idle while an Env runs a judge. Release its permit
+            # so a one-agent episode can enter the nested judge rollout.
+            if self._gate is not None:
+                self._gate.release()
+            try:
+                try:
+                    await callback(trace, runtime)
+                finally:
+                    _scoring_runs.reset(token)
+                    if self._gate is not None:
+                        await run_shielded(self._gate.acquire())
+            except BaseException:
+                for agent, child in runs:
+                    agent._completed[:] = [
+                        t for t in agent._completed if t is not child
+                    ]
+                    if agent._on_discard is not None:
+                        agent._on_discard(child)
+                    if child.usage is not None:
+                        trace.extra_usage.append(child.usage)
+                    trace.extra_usage.extend(child.extra_usage)
+                raise
+            else:
+                if parent_runs is not None:
+                    parent_runs.extend(runs)
+
+        return score
+
     async def run(
         self,
         task: Task,
@@ -929,6 +977,7 @@ class _EpisodeAgent(Agent):
         tools: Mapping[str, SharedToolServer] | None = None,
         on_trace: Callable[[Trace], None] | None = None,
         collect_artifacts: bool = False,
+        on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None,
     ) -> Trace:
         async with self._gate or nullcontext():
             trace = await super().run(
@@ -937,6 +986,7 @@ class _EpisodeAgent(Agent):
                 tools=tools if tools is not None else self._shared_for(task),
                 on_trace=self._watch(on_trace),
                 collect_artifacts=collect_artifacts,
+                on_score=self._scoring_callback(on_score),
             )
         self._completed.append(trace)
         return trace

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/verifiers/v1/cli/debug.py

@@ -26,7 +26,7 @@ from verifiers.v1.cli.resolve import (
     with_positional_taskset,
 )
 from verifiers.v1.configs.cli.debug import DebugConfig
-from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
+from verifiers.v1.runtimes import ProgramResult, Runtime, make_backend
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask
@@ -216,20 +216,21 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         "reason": "not_run",
     }
     cancelled = False
-    runtime = make_runtime(
+    backend = make_backend(
         resolve_runtime_config(config.runtime, task),
         name=f"debug-{task.data.idx}-{uuid4().hex[:8]}",
     )
-    trace.agent.runtime = runtime.info
+    trace.agent.runtime = backend.info
     setup_timeout = (
         config.timeout.setup
         if config.timeout.setup is not None
         else task.data.timeout.setup
     )
     try:
-        runtime.env = dict(task.runtime_env())
+        backend.env = dict(task.runtime_env())
         trace.timing.boot.start = time.time()
-        await runtime.start()
+        await backend.start()
+        runtime = backend.bind()
         now = time.time()
         trace.timing.boot.end = now
         trace.timing.setup.start = now
@@ -237,7 +238,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
             setup_timeout,
         )
-        await runtime.prepare_execution([])
+        await backend.prepare_execution([])
         trace.timing.setup.end = time.time()
 
         trace.timing.agent.start = time.time()
@@ -256,7 +257,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         trace.split_agent_time()
         trace.info["debug"] = debug
         try:
-            await runtime.stop()
+            await backend.stop()
         except asyncio.CancelledError:
             # a task cancellation delivered mid-stop would abort before the caller can
             # persist the trace — absorb it here; the caller re-raises after appending

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/verifiers/v1/cli/validate.py

@@ -31,7 +31,7 @@ from verifiers.v1.cli.resolve import (
 from verifiers.v1.cli.resume import distribute
 from verifiers.v1.configs.cli.validate import ValidateConfig
 from verifiers.v1.errors import TaskError, boundary
-from verifiers.v1.runtimes import make_runtime
+from verifiers.v1.runtimes import make_backend
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
@@ -266,7 +266,7 @@ CHECKS = {"setup": _setup_only, "gold": _gold, "noop": _score_untouched}
 
 async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow:
     start = time.time()
-    runtime = make_runtime(
+    backend = make_backend(
         resolve_runtime_config(config.runtime, task),
         name=f"validate-{mode}-{task.data.idx}-{uuid4().hex[:8]}",
     )
@@ -279,7 +279,7 @@ async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow
     exc = None
     trace: Trace | None = None
     try:
-        runtime.env = dict(task.runtime_env())
+        backend.env = dict(task.runtime_env())
         trace = Trace(
             task=TraceTask(
                 type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
@@ -292,7 +292,8 @@ async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow
                 trainable=False,
             ),
         )
-        await runtime.start()
+        await backend.start()
+        runtime = backend.bind()
         await asyncio.wait_for(
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
             setup_timeout,
@@ -304,7 +305,7 @@ async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow
         exc = e
     finally:
         try:
-            await runtime.stop()
+            await backend.stop()
         except Exception:
             logger.warning(
                 "runtime teardown failed (task %s)", task.data.idx, exc_info=True

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/verifiers/v1/configs/agent.py

@@ -2,13 +2,22 @@
 
 from typing import Any
 
-from pydantic import BaseModel, Field, SerializeAsAny, field_serializer, model_validator
+from pydantic import (
+    BaseModel,
+    Field,
+    SerializationInfo,
+    SerializeAsAny,
+    field_serializer,
+    model_validator,
+)
 from pydantic_config import BaseConfig
 
 from verifiers.v1.clients import ClientConfig
 from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.scoring import ScoringConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
+from verifiers.v1.runtimes.deployment import DeploymentConfig
 from verifiers.v1.types import SamplingConfig
 from verifiers.v1.utils.generic import merge_defaults
 
@@ -25,16 +34,23 @@ class TimeoutConfig(BaseConfig):
     finalize: float | None = Field(None, gt=0)
     """Timeout (in seconds) for the task + harness finalize hooks."""
     scoring: float | None = Field(None, gt=0)
-    """Timeout (in seconds) for the task + harness metrics + scoring hooks."""
+    """Combined budget (in seconds) for harness metrics and task scoring, including
+    isolated verifier setup and retries. Finalization has its own budget."""
 
 
 class AgentConfig(BaseConfig):
     harness: SerializeAsAny[HarnessConfig] | None = None
     """The agent's program (None = the taskset's default harness)."""
     runtime: RuntimeConfig = PrimeConfig()
-    """Runtime for the harness program — the policy each run provisions its box
+    """Runtime configuration for the harness program — the policy each run provisions its box
     from; tool servers choose their placement separately."""
 
+    scoring: ScoringConfig = ScoringConfig()
+    """Scoring overrides; unset fields inherit the task’s scoring defaults."""
+
+    deployment: DeploymentConfig = DeploymentConfig()
+    """Workspace and service placement, including trust for local Compose tasks."""
+
     model: str | None = None
     """Model id (None = the run's model, i.e. the policy under evaluation/training)."""
     client: ClientConfig | None = None
@@ -57,6 +73,14 @@ class AgentConfig(BaseConfig):
     network_notice: bool = True
     """Include the runtime's network restrictions in the agent's prompt."""
 
+    @field_serializer("scoring")
+    def _scoring_overrides(
+        self, scoring: ScoringConfig, info: SerializationInfo
+    ) -> dict:
+        # Preserve unset fields across env-server and trace roundtrips so they
+        # continue to inherit from the task instead of becoming explicit overrides.
+        return scoring.model_dump(mode=info.mode, exclude_unset=True)
+
     @field_serializer("sampling")
     def _sampling_overrides(self, sampling: SamplingConfig | None) -> dict | None:
         # An unresolved agent carries only overrides across env-server boundaries.

{verifiers-0.4.1.dev14 → verifiers-0.4.1.dev19}/verifiers/v1/configs/retries.py

@@ -1,4 +1,4 @@
-"""Whole-rollout retry rules — each agent's own and the env's episode fallback."""
+"""Retry rules shared by episode, agent rollout, and isolated scoring boundaries."""
 
 import re
 from typing import Annotated, Literal
@@ -36,7 +36,7 @@ class RetryRule(BaseConfig):
 
 
 class RetryConfig(BaseConfig):
-    """Ordered rules for opt-in whole-rollout retries.
+    """Ordered rules for retries at the owning config’s execution boundary.
 
     Fields within a rule are ANDed. An exhausted rule never falls through to a
     later rule for that error; another captured error can still trigger a retry.