verifiers 0.2.2.dev15__tar.gz → 0.2.2.dev17__tar.gz

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev15
+Version: 0.2.2.dev17
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/tests/v1/test_configs.py

@@ -24,4 +24,4 @@ CONFIGS = sorted(
 def test_eval_config_parses(path: Path) -> None:
     config = EvalConfig.model_validate(tomllib.load(path.open("rb")))
     # resolved to a v1 taskset or a v0 env id
-    assert (config.env.taskset is not None and config.env.taskset.id) or config.id
+    assert config.env.taskset.id or config.id

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/agent.py

@@ -26,6 +26,7 @@ from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.retries import RetryConfig, backoff, trace_should_retry
 from verifiers.v1.rollout import RolloutRun
 from verifiers.v1.runtimes import (
+    DockerConfig,
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
@@ -93,11 +94,37 @@ class AgentConfig(BaseConfig):
         return data
 
 
-def _check_borrowed_placement(task: Task, runtime: Runtime) -> None:
+def _check_borrowed_placement(
+    task: Task, runtime: Runtime, base_config: RuntimeConfig
+) -> None:
     """A borrowed box is never re-provisioned, so a task's placement fields can't
-    be honored. A task `image` on a subprocess box raises (a wiring bug — it goes
-    to the caller, not the trace); a container box whose image differs only warns,
-    since placing a run into an existing world is the point of borrowing."""
+    be honored. Reject requirements that cannot be applied to the running box; an
+    image mismatch on a container only warns, since sharing its world is the point."""
+    task_policy = "*" not in task.data.network_allow or bool(task.data.network_block)
+    base_docker = base_config if isinstance(base_config, DockerConfig) else None
+    if task_policy or (base_docker is not None and base_docker.network_isolated):
+        config = runtime.config
+        if not isinstance(config, DockerConfig):
+            raise ValueError(
+                f"task {task.data.idx!r} requires a Docker URL network policy, but "
+                f"borrowed runtime {runtime.name!r} is not Docker-backed; use "
+                "agent.provision(task)"
+            )
+        policy_base = (
+            base_docker if base_docker is not None else DockerConfig(allow=["*"])
+        )
+        expected = resolve_runtime_config(policy_base, task)
+        assert isinstance(expected, DockerConfig)
+        # Do not inherit extra destinations from a box provisioned for another task.
+        if set(config.allow) != set(expected.allow) or set(config.block) != set(
+            expected.block
+        ):
+            raise ValueError(
+                f"task {task.data.idx!r} requires allow={expected.allow!r} and "
+                f"block={expected.block!r}, but borrowed runtime {runtime.name!r} "
+                f"has allow={config.allow!r} and block={config.block!r}; use "
+                "agent.provision(task)"
+            )
     if task.data.image is None:
         return
     if isinstance(runtime.config, SubprocessConfig):
@@ -299,7 +326,7 @@ class Agent:
     ) -> dict:
         """Resolve one run's runtime config, pairing checks, timeouts, interception."""
         if runtime is not None:
-            _check_borrowed_placement(task, runtime)
+            _check_borrowed_placement(task, runtime, self.runtime_config)
             runtime_config = runtime.config
             run_is_local = runtime.is_local
         else:

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/cli/dashboard/eval.py

@@ -232,7 +232,7 @@ def Overview(config: EvalConfig) -> Table:
     grid.add_column()
     seats = config.env.agent_harnesses()
     taskset = config.env.taskset
-    env_label = taskset.name if taskset is not None else "no taskset"
+    env_label = taskset.name if taskset.id else "no taskset"
     if config.env.id:
         env_label = f"{env_name(config.env.id)}+{env_label}"
     # One seat story when every seat resolves the same way (the common case); one
@@ -254,9 +254,7 @@ def Overview(config: EvalConfig) -> Table:
     # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be
     # parsed as styling and dropped. `id` is in the `env` row; harness `runtime.type` too (hidden
     # here), but only for the harness — `taskset.task.tools.runtime.type` has no other display.
-    if taskset is not None and (
-        taskset_over := overrides(taskset, skip=frozenset({"id"}))
-    ):
+    if taskset_over := overrides(taskset, skip=frozenset({"id"})):
         grid.add_row("taskset", escape("  ·  ".join(taskset_over)))
     for role, h in seats.items():
         if harness_over := overrides(h, skip=frozenset({"id", "runtime.type"})):

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/cli/debug.py

@@ -230,6 +230,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
             setup_timeout,
         )
+        await runtime.prepare_execution([])
         trace.timing.setup.end = time.time()
 
         trace.timing.generation.start = time.time()

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/cli/eval/resume.py

@@ -4,13 +4,20 @@
 flags) and writes back into the same dir. `load` keeps the good saved rollouts and
 re-runs what's owed: missing rollouts (never written) and errored ones (dropped and
 redone).
+
+A saved rollout is matched to a selected task by content: `task_key` hashes the
+task's wire data. Tasks with identical data are interchangeable, a task whose data
+changed since the interrupted run re-runs, and nothing depends on `data.idx`. The
+legacy (v0) bridge still matches by row index (`key_of`).
 """
 
+import hashlib
 import json
 import tomllib
-from collections import defaultdict
-from collections.abc import Callable
+from collections import Counter, defaultdict
+from collections.abc import Callable, Hashable, Mapping
 from pathlib import Path
+from typing import TypeVar
 
 from pydantic_core import from_json
 
@@ -19,6 +26,30 @@ from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.trace import WireTrace
 
+K = TypeVar("K", bound=Hashable)
+
+
+def task_key(data: Mapping) -> str:
+    """Content identity of one task's wire data — an `exclude_none` dump, the shape
+    saved rows already have on disk. `sort_keys` so field order can't split identity."""
+    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
+
+
+def distribute(
+    selected_keys: list[K], owed: dict[K, int], num_rollouts: int
+) -> list[int]:
+    """Spread each key's owed rollouts over its selection instances, in order —
+    content-identical tasks are interchangeable, so any instance can absorb the
+    debt (capped at `num_rollouts` each). Returns one count per selection."""
+    remaining = dict(owed)
+    counts: list[int] = []
+    for key in selected_keys:
+        take = min(num_rollouts, remaining.get(key, 0))
+        if take:
+            remaining[key] -= take
+        counts.append(take)
+    return counts
+
 
 def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
     """Pull `--resume <dir>` / `--resume=<dir>` out of argv, returning (dir, the other args).
@@ -51,25 +82,27 @@ def load_resume_config(resume_dir: Path) -> EvalConfig:
 
 def load(
     resume_dir: Path,
-    selected_idxs: list[int],
+    selected_keys: list[K],
     num_rollouts: int,
     complete: Callable[[Episode], bool] | None = None,
     *,
     whole_task: bool = False,
-) -> tuple[list[Episode], dict[int, int]]:
-    """Load the good saved rollouts as finished episodes and diff them against the
-    run's target: returns (kept episodes, rollouts owed per task idx). A rollout is
-    kept or redone as a unit — the episode — so a multi-trace rollout interrupted
-    mid-write is simply owed again. `complete` is the environment's keep-verdict
-    (`Env.complete`); without it (the server path) the default is
-    `episode.ok`, so an errored rollout is dropped and re-run. `whole_task` redoes
-    a partially-kept task as a unit — the legacy group-scored path, where
-    `run_group` always serves the full n. Rewrites `traces.jsonl` to just the kept
-    rows via a temp file + atomic rename, so an interrupted resume can't corrupt
-    the prior results; the resumed rollouts then append. Pre-episode files (one
-    bare trace per line) load each trace as a single-trace episode."""
+    key_of: Callable[[Mapping], K] | None = None,
+) -> tuple[list[Episode], dict[K, int]]:
+    """Load the good saved rollouts and diff them against the run's target: returns
+    (kept episodes, rollouts owed per task key). `selected_keys` is one key per
+    selected task (duplicates allowed — a key selected k times is owed up to
+    `k * num_rollouts`; spread back over the tasks with `distribute`). `key_of` maps
+    a saved row's task data to its key (default `task_key`; the legacy bridge uses
+    row indices). `complete` is the keep-verdict (default `episode.ok`); `whole_task`
+    redoes a partially-kept task whole (legacy group scoring). Rewrites
+    `traces.jsonl` to the kept rows via a temp file + atomic rename; a torn or
+    malformed row is owed again, never a crash."""
     path = resume_dir / TRACES_FILE
-    selected = set(selected_idxs)
+    targets = {
+        key: count * num_rollouts for key, count in Counter(selected_keys).items()
+    }
+    keyed = key_of if key_of is not None else task_key
 
     def parse(row: dict) -> Episode:
         if sniff_episode(row):
@@ -77,7 +110,7 @@ def load(
         return Episode.of(WireTrace.model_validate(row))
 
     verdict = complete if complete is not None else (lambda episode: episode.ok)
-    good: dict[int, list[tuple[bytes, Episode]]] = defaultdict(list)
+    good: dict[K, list[tuple[bytes, Episode]]] = defaultdict(list)
     if path.exists():
         with path.open("rb") as results:
             for line in results:
@@ -89,16 +122,16 @@ def load(
                     except ValueError:
                         row = json.loads(line)
                     # The task rides each trace; a traceless record (a failure
-                    # before any trace minted) has no idx and is owed again.
+                    # before any trace minted) has no task and is owed again.
                     if sniff_episode(row):
-                        idx = row["traces"][0]["task"]["data"]["idx"]
+                        key = keyed(row["traces"][0]["task"]["data"])
                     else:
-                        idx = row["task"]["data"]["idx"]
+                        key = keyed(row["task"]["data"])
                 except (ValueError, KeyError, IndexError, TypeError):
                     # A torn final line (the run died mid-write) or a foreign shape
                     # is not a keepable rollout — it's owed again, never a crash.
                     continue
-                if idx not in selected or len(good[idx]) >= num_rollouts:
+                if key not in targets or len(good[key]) >= targets[key]:
                     continue
                 try:
                     episode = parse(row)
@@ -106,20 +139,20 @@ def load(
                         continue
                 except Exception:  # malformed row: redo it
                     continue
-                good[idx].append(
+                good[key].append(
                     (line if line.endswith(b"\n") else line + b"\n", episode)
                 )
     keep: list[bytes] = []
     episodes: list[Episode] = []
-    owed: dict[int, int] = {}
-    for idx in selected_idxs:
-        rows = good.get(idx, [])
-        if whole_task and len(rows) < num_rollouts:
+    owed: dict[K, int] = {}
+    for key, target in targets.items():
+        rows = good.get(key, [])
+        if whole_task and len(rows) < target:
             rows = []  # a partial unit redoes whole — its kept rows are dropped
         keep.extend(line for line, _ in rows)
         episodes.extend(episode for _, episode in rows)
-        if missing := num_rollouts - len(rows):
-            owed[idx] = missing
+        if missing := target - len(rows):
+            owed[key] = missing
     tmp = path.with_suffix(".jsonl.tmp")
     tmp.write_bytes(b"".join(keep))
     tmp.replace(path)

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/cli/eval/runner.py

@@ -32,21 +32,25 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
     )
     out = output_path(config)
-    owed: dict[str, int] | None = None
+    # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts.
+    plan = [(task, config.num_rollouts) for task in tasks]
     # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run.
     finished: list[Episode] = []
     if config.resume is not None:
-        finished, owed = resume.load(
-            out, [t.data.idx for t in tasks], config.num_rollouts, env.complete
-        )
+        keys = [
+            resume.task_key(t.data.model_dump(mode="json", exclude_none=True))
+            for t in tasks
+        ]
+        finished, owed = resume.load(out, keys, config.num_rollouts, env.complete)
         if not owed:  # already complete - report it and exit successfully
             print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts))
             raise SystemExit(0)
-        tasks = [task for task in tasks if owed.get(task.data.idx)]
+        counts = resume.distribute(keys, owed, config.num_rollouts)
+        plan = [(task, n) for task, n in zip(tasks, counts) if n]
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
             out,
-            len(tasks),
+            len(plan),
             sum(owed.values()),
         )
     else:
@@ -70,13 +74,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     # Serving resources (shared tool servers, interception) come up once for the
     # run; plan slots inside so the env's agents borrow them.
     async with env.serving():
-        planned = [
-            slot
-            for task in tasks
-            for slot in env.slots(
-                task, n=owed[task.data.idx] if owed else config.num_rollouts
-            )
-        ]
+        planned = [slot for task, n in plan for slot in env.slots(task, n=n)]
         slots = [RunSlot.finished(episode) for episode in finished] + planned
         push_state = None
         if config.push and config.rich:
@@ -123,6 +121,15 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         if legacy
         else {"config_data": env_config_data(config.env)}  # picklable across the spawn
     )
+    tasks = []
+    if not legacy:
+        from verifiers.v1.loaders import load_taskset
+
+        # The client owns the taskset: load it here, once — the server (and its pool
+        # workers) never load data, they rebuild each dispatched task from its request.
+        tasks = load_taskset(config.env.taskset).select(
+            config.num_tasks, config.shuffle
+        )
     # Spawned processes inherit no logging — hand them the main process's setup so
     # their rollout logs land in the output dir.
     level = "DEBUG" if config.verbose else "INFO"
@@ -151,48 +158,57 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         address = await asyncio.to_thread(address_queue.get, timeout=600)
         client = EnvClient(address=address)
         await client.wait_for_server_startup(timeout=600)
-        info = await client.info()
-        # Only a legacy (v0) env group-scores; a v1 env scores siblings in its own
-        # rollout.
-        group_scored = info.requires_group_scoring
-        if info.num_tasks is None:  # infinite taskset - the run must be bounded
-            if config.num_tasks is None:
-                raise ValueError(
-                    f"{config.env_id} is infinite - bound the run with -n/--num-tasks"
-                )
-            if config.shuffle:
-                logger.warning(
-                    "shuffle is a no-op on an infinite taskset - "
-                    "taking the first %d generated tasks",
-                    config.num_tasks,
-                )
-            idxs = list(range(config.num_tasks))
-        else:
+        # A v1 run dispatches — and resumes — tasks by content: the client owns them,
+        # and `resume.task_key` is their identity. Only the legacy bridge is addressed
+        # by dataset row (its dataset lives server-side, reported via `info`), and
+        # only a legacy env group-scores; a v1 env scores siblings in its own rollout.
+        if legacy:
+            info = await client.info()
+            group_scored = info.requires_group_scoring
             idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
+            plan = [({"task_idx": idx}, config.num_rollouts) for idx in idxs]
+        else:
+            group_scored = False
+            plan = [
+                ({"task_data": task.data.model_dump(mode="json")}, config.num_rollouts)
+                for task in tasks
+            ]
         out = output_path(config)
         finished: list[Episode] = []
         if config.resume is not None:
-            # (legacy only) a group is served and scored together, so a partially-kept
-            # task redoes as a whole group — whole_task drops its kept rows.
-            finished, owed = resume.load(
-                out, idxs, config.num_rollouts, whole_task=group_scored
-            )
+            if legacy:
+                # A group is served and scored together, so a partially-kept task
+                # redoes as a whole group — whole_task drops its kept rows.
+                finished, owed = resume.load(
+                    out,
+                    idxs,
+                    config.num_rollouts,
+                    whole_task=group_scored,
+                    key_of=lambda data: data.get("idx"),
+                )
+                counts = resume.distribute(idxs, owed, config.num_rollouts)
+            else:
+                keys = [
+                    resume.task_key(t.data.model_dump(mode="json", exclude_none=True))
+                    for t in tasks
+                ]
+                finished, owed = resume.load(out, keys, config.num_rollouts)
+                counts = resume.distribute(keys, owed, config.num_rollouts)
             if not owed:  # already complete - report it and exit successfully
-                print(resume.nothing_to_resume_msg(out, len(idxs), config.num_rollouts))
+                print(resume.nothing_to_resume_msg(out, len(plan), config.num_rollouts))
                 raise SystemExit(0)
-            idxs = [idx for idx in idxs if owed.get(idx)]
+            plan = [(payload, n) for (payload, _), n in zip(plan, counts) if n]
             logger.info(
                 "resuming %s: %d task(s), %d rollout(s) owed",
                 out,
-                len(idxs),
+                len(plan),
                 sum(owed.values()),
             )
         else:
-            owed = {idx: config.num_rollouts for idx in idxs}
             save_config(config, out)
             logger.info(
                 "running %dx%d rollouts via the env-server %s pool on %s",
-                len(idxs),
+                len(plan),
                 config.num_rollouts,
                 config.pool.type,
                 config.model,
@@ -224,13 +240,13 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 records.append(Episode.of(trace))
             return records
 
-        async def run_unit(idx: int) -> list[Episode]:
+        async def run_unit(payload: dict) -> list[Episode]:
             async with semaphore or contextlib.nullcontext():
                 episode = await client.run(
-                    task_idx=idx,
                     client=config.client,
                     model=config.model,
                     sampling=config.sampling,
+                    **payload,
                 )
             for trace in episode.traces:
                 trace.stamp(EvalRunInfo(id=config.uuid))
@@ -238,12 +254,12 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
             return [episode]
 
         # A group-scored legacy task runs its rollouts together (one `run_group`
-        # request, one worker); otherwise each rollout is its own `run_rollout`
-        # request, dispatched least-busy across workers.
+        # request, one worker); otherwise each rollout is its own `run` request,
+        # dispatched least-busy across workers.
         units = (
-            [run_group_unit(i) for i in idxs]
+            [run_group_unit(payload["task_idx"]) for payload, _ in plan]
             if group_scored
-            else [run_unit(i) for i in idxs for _ in range(owed[i])]
+            else [run_unit(payload) for payload, n in plan for _ in range(n)]
         )
         results = await asyncio.gather(*units)
         await client.close()

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/cli/gepa.py

@@ -66,7 +66,7 @@ def main(argv: list[str] | None = None) -> None:
     # Refuse multi-agent before the dry-run return, so --dry-run can't write a
     # config the real invocation would reject.
     env_cls = vf.environment_class(
-        config.env.taskset.id if config.env.taskset is not None else "",
+        config.env.taskset.id,
         config.env.id,
     )
     if not issubclass(env_cls, vf.SingleAgentEnv):

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/cli/output.py

@@ -36,8 +36,8 @@ def output_path(config: EvalConfig) -> Path:
     if config.output_dir is not None:
         return config.output_dir
     taskset = config.env.taskset
-    env = taskset.name if taskset is not None else "no-taskset"
-    if taskset is not None and taskset.id and config.env.id:
+    env = taskset.name if taskset.id else "no-taskset"
+    if taskset.id and config.env.id:
         # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k-v1` run must
         # not share a parent dir with a plain `gsm8k-v1` one.
         env = f"{env_name(config.env.id)}+{env}"

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/configs/env.py

@@ -138,9 +138,7 @@ class EnvServerConfig(BaseConfig):
     @property
     def is_legacy(self) -> bool:
         """A v0/legacy env (run via the bridge): a legacy `id` is set and no v1 taskset."""
-        return self.id is not None and (
-            self.env.taskset is None or not self.env.taskset.id
-        )
+        return self.id is not None and not self.env.taskset.id
 
     @property
     def env_id(self) -> str:
@@ -152,7 +150,7 @@ class EnvServerConfig(BaseConfig):
     def _refuse_legacy_id_with_taskset(self):
         """A legacy `id` next to a v1 `env.taskset` would be silently inert
         (`is_legacy` is False and the v0 env never loads); refuse the mix."""
-        if self.id is not None and self.env.taskset is not None and self.env.taskset.id:
+        if self.id is not None and self.env.taskset.id:
             raise ValueError(
                 f"--id {self.id!r} is the legacy (v0) env id and can't combine with "
                 f"the v1 taskset {self.env.taskset.id!r}. Pairing an env with a "

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/env.py

@@ -31,7 +31,7 @@ from verifiers.v1.retries import RetryConfig, run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.errors import EnvError, boundary
 from verifiers.v1.task import Task, resolve_server_config
-from verifiers.v1.taskset import Taskset, TasksetConfig
+from verifiers.v1.taskset import TasksetConfig
 from verifiers.v1.episode import Episode
 from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.generic import deep_merge, generic_type
@@ -65,9 +65,9 @@ class EnvConfig(BaseConfig):
     """Which `Env` runs. Empty = the taskset's own, else `SingleAgentEnv`; set
     to pair a reusable env with any taskset (an explicit id wins over the bundled)."""
     # SerializeAsAny: the env-server wire needs the resolved subclass's fields.
-    taskset: SerializeAsAny[TasksetConfig] | None = None
+    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
     """The seed taskset — the rows every rollout starts from (`--env.taskset.id`).
-    None only for an env that mints its tasks without a dataset."""
+    The id stays empty only for a legacy (v0) run, which sets the top-level `id`."""
     timeout: TimeoutConfig = TimeoutConfig()
     retries: RetryConfig = RetryConfig()
     """Whole-EPISODE retries — the coarse fallback for faults no agent owns; a
@@ -81,17 +81,14 @@ class EnvConfig(BaseConfig):
     @property
     def env_id(self) -> str:
         """The taskset id, prefixed by the paired env id (`best-of-n+gsm8k-v1`)."""
-        taskset_id = self.taskset.id if self.taskset is not None else ""
-        if taskset_id and self.id:
-            return f"{self.id}+{taskset_id}"
-        return taskset_id or self.id
+        if self.taskset.id and self.id:
+            return f"{self.id}+{self.taskset.id}"
+        return self.taskset.id or self.id
 
     def agent_harnesses(self) -> dict[str, HarnessConfig]:
         """Each declared role's resolved harness config (pin, else the taskset's
         default) — known without constructing the env."""
-        default = default_agent_harness(
-            self.taskset.id if self.taskset is not None else ""
-        )
+        default = default_agent_harness(self.taskset.id)
         return {
             name: cfg.harness if cfg.harness is not None else default
             for name, cfg in _declared_agent_configs(self).items()
@@ -239,7 +236,7 @@ class Env(ABC, Generic[ConfigT]):
                 f"{config_cls.__name__}(...) explicitly"
             )
         self.config: ConfigT = config
-        if config.taskset is None:
+        if not config.taskset.id:
             raise ValueError(
                 f"{type(self).__name__} needs a seed taskset — every rollout starts "
                 "from one of its tasks: set --env.taskset.id (or the positional "
@@ -247,7 +244,7 @@ class Env(ABC, Generic[ConfigT]):
             )
         self.taskset = load_taskset(config.taskset)
         self._default_harness = default_agent_harness(config.taskset.id)
-        task_cls = generic_type(type(self.taskset), Task, origin=Taskset) or Task
+        task_cls = type(self.taskset).task_type()
         self._task_cls: type[Task] = task_cls
         self._agent_specs: dict[str, AgentConfig] = _declared_agent_configs(self.config)
         if not self._agent_specs:
@@ -520,7 +517,7 @@ class Env(ABC, Generic[ConfigT]):
         """`requires_tunnel` over the consumers known before any rollout: role
         runtimes, live `shared` servers, and the task class's tool/user servers;
         a class overriding `server_config` conservatively counts as remote."""
-        task_cls = generic_type(type(self.taskset), Task, origin=Taskset) or Task
+        task_cls = type(self.taskset).task_type()
         server_classes = [*task_cls.tools, *([task_cls.user] if task_cls.user else [])]
         if server_classes and task_cls.server_config is not Task.server_config:
             return True

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/interception/__init__.py

@@ -34,13 +34,10 @@ def requires_tunnel(
     server_configs: Iterable[BaseConfig] = (),
     shared: "Iterable[SharedToolServer]" = (),
 ) -> bool:
-    """Whether the interception must be exposed via a tunnel — some consumer is off the
-    host network: the harness itself, a live `shared` server in a remote runtime, or a
-    tool/user server config placing one there (each reaches the `/state` channel from
-    its own runtime). Skipped as non-consumers: a `colocated` server (shares the
-    harness's runtime, covered by `harness_is_local`), a config-`url` server (external —
-    it connects out), and an `external` shared server (outside the state machinery
-    entirely). False means every consumer reaches the server at localhost."""
+    """Whether interception needs a public tunnel because some consumer cannot use a
+    host-local URL: the harness itself, a live `shared` server in a remote runtime, or a
+    tool/user server placed there. Colocated servers, configured URLs, and external shared
+    servers add no consumer. False means local URL translation is sufficient."""
     if not harness_is_local:
         return True
     if any(not s.external and not s.local for s in shared):
@@ -57,10 +54,8 @@ def make_interception(
     config: InterceptionConfig, *, requires_tunnel: bool
 ) -> Interception:
     """The interception for a config, picked by type (the host-side counterpart to
-    `make_runtime`). With `requires_tunnel` (some consumer is off the host network — see
-    the `requires_tunnel` util) each server is exposed via its configured tunnel; without
-    it they get none and are reached at localhost. The caller computes it (see
-    `Env._requires_tunnel`)."""
+    `make_runtime`). With `requires_tunnel`, each server is exposed through its configured
+    tunnel; otherwise it remains on host loopback. The caller computes this requirement."""
     if isinstance(config, InterceptionServerConfig):
         return InterceptionServer(config, requires_tunnel)
     if isinstance(config, StaticInterceptionPoolConfig):

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/interception/server.py

@@ -121,11 +121,9 @@ class InterceptionServerConfig(BaseInterceptionConfig):
 
 class InterceptionServer(Interception):
     """A server that proxies model calls for one or more rollouts — and is itself the
-    single-server `Interception` (the pools compose several of these). With
-    `requires_tunnel` (some consumer is off the host network) it mints its configured
-    tunnel; on `start` it then binds where the tunnel says (`bind_host`/`bind_port`) and
-    sets `base_url` — the one URL every consumer reaches it at — to the tunnel's public
-    URL. Without, every consumer is on the host network: it binds loopback, tunnel-free."""
+    single-server `Interception` (the pools compose several of these). When a consumer
+    needs a public URL, it mints the configured tunnel and binds where that tunnel says;
+    otherwise it stays on host loopback."""
 
     def __init__(
         self,
@@ -203,9 +201,8 @@ class InterceptionServer(Interception):
         self.runner = web.AppRunner(app)
         await self.runner.setup()
         self.stack.push_async_callback(self.runner.cleanup)
-        # No tunnel → every consumer shares the host network: bind loopback on any ephemeral
-        # port. Otherwise the tunnel says where to bind for it to reach the port, and
-        # `expose` publishes it.
+        # Without a tunnel, local URL translation reaches an ephemeral loopback port.
+        # Otherwise the tunnel determines the bind address and publishes it.
         if self.tunnel is None:
             self.host, bind_port = "127.0.0.1", 0
         else:

{verifiers-0.2.2.dev15 → verifiers-0.2.2.dev17}/verifiers/v1/interception/tunnel/base.py

@@ -1,12 +1,9 @@
 """The Tunnel contract: expose a host interception port to a remote consumer.
 
-The interception server runs on the host. A *local* consumer (host network) reaches it at
-localhost — no tunnel, whatever the config. A *remote* one needs the port published outward:
-that's a `Tunnel`. It says where the server must bind for the tunnel to reach it
-(`bind_host`/`bind_port`) and `expose`s the bound port as a public URL. So a tunnel knows
-nothing about locality — the caller (`InterceptionServer.start`) uses it only in the remote
-case. The host-side counterpart to a `Runtime`: `Runtime.expose` publishes a port *inside* a
-sandbox; `Tunnel.expose` a *host* port.
+The interception server runs on the host. Local runtimes use a host-local URL directly or
+through their runtime translation; remote runtimes need the port published outward. A
+`Tunnel` supplies the bind address and exposes that host port as a public URL. It is the
+host-side counterpart to `Runtime.expose`, which publishes a port inside a sandbox.
 """
 
 import contextlib