verifiers 0.2.2.dev16__tar.gz → 0.2.2.dev18__tar.gz

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev16
+Version: 0.2.2.dev18
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/tests/v1/test_configs.py

@@ -24,4 +24,4 @@ CONFIGS = sorted(
 def test_eval_config_parses(path: Path) -> None:
     config = EvalConfig.model_validate(tomllib.load(path.open("rb")))
     # resolved to a v1 taskset or a v0 env id
-    assert (config.env.taskset is not None and config.env.taskset.id) or config.id
+    assert config.env.taskset.id or config.id

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/tests/v1/test_e2e.py

@@ -15,26 +15,28 @@ def _pair(a: str, b: str, id: str, *extra_marks):
     return pytest.param(a, b, marks=[*marks, *extra_marks], id=id)
 
 
-# harness x harness runtime: every harness once, both local runtimes hit, one remote row
-# per provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
+# harness x harness runtime: every harness once, both local runtimes hit (subprocess
+# only carries the in-house loops — the rest NEEDS_CONTAINER), one remote row per
+# provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
 # task) — test_agentic covers them.
 CHAT_PLACEMENTS = [
     _pair("null", "subprocess", "null-harness-in-subprocess"),
     _pair("bash", "docker", "bash-harness-in-docker"),
-    _pair("rlm", "subprocess", "rlm-harness-in-subprocess"),
+    _pair("rlm", "docker", "rlm-harness-in-docker"),
     _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # harness x harness runtime for the shell task: every coding agent once (null is a chat
-# loop with no shell), both local runtimes hit, one remote row per provider.
+# loop with no shell), both local runtimes hit (subprocess only carries bash), one
+# remote row per provider.
 AGENTIC_PLACEMENTS = [
     _pair("bash", "subprocess", "bash-harness-in-subprocess"),
     _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "subprocess", "kimi-code-harness-in-subprocess"),
+    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("codex", "docker", "codex-harness-in-docker"),
-    _pair("claude-code", "subprocess", "claude-code-harness-in-subprocess"),
+    _pair("claude-code", "docker", "claude-code-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/cli/dashboard/eval.py

@@ -232,7 +232,7 @@ def Overview(config: EvalConfig) -> Table:
     grid.add_column()
     seats = config.env.agent_harnesses()
     taskset = config.env.taskset
-    env_label = taskset.name if taskset is not None else "no taskset"
+    env_label = taskset.name if taskset.id else "no taskset"
     if config.env.id:
         env_label = f"{env_name(config.env.id)}+{env_label}"
     # One seat story when every seat resolves the same way (the common case); one
@@ -254,9 +254,7 @@ def Overview(config: EvalConfig) -> Table:
     # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be
     # parsed as styling and dropped. `id` is in the `env` row; harness `runtime.type` too (hidden
     # here), but only for the harness — `taskset.task.tools.runtime.type` has no other display.
-    if taskset is not None and (
-        taskset_over := overrides(taskset, skip=frozenset({"id"}))
-    ):
+    if taskset_over := overrides(taskset, skip=frozenset({"id"})):
         grid.add_row("taskset", escape("  ·  ".join(taskset_over)))
     for role, h in seats.items():
         if harness_over := overrides(h, skip=frozenset({"id", "runtime.type"})):

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/cli/eval/resume.py

@@ -4,13 +4,20 @@
 flags) and writes back into the same dir. `load` keeps the good saved rollouts and
 re-runs what's owed: missing rollouts (never written) and errored ones (dropped and
 redone).
+
+A saved rollout is matched to a selected task by content: `task_key` hashes the
+task's wire data. Tasks with identical data are interchangeable, a task whose data
+changed since the interrupted run re-runs, and nothing depends on `data.idx`. The
+legacy (v0) bridge still matches by row index (`key_of`).
 """
 
+import hashlib
 import json
 import tomllib
-from collections import defaultdict
-from collections.abc import Callable
+from collections import Counter, defaultdict
+from collections.abc import Callable, Hashable, Mapping
 from pathlib import Path
+from typing import TypeVar
 
 from pydantic_core import from_json
 
@@ -19,6 +26,30 @@ from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.trace import WireTrace
 
+K = TypeVar("K", bound=Hashable)
+
+
+def task_key(data: Mapping) -> str:
+    """Content identity of one task's wire data — an `exclude_none` dump, the shape
+    saved rows already have on disk. `sort_keys` so field order can't split identity."""
+    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
+
+
+def distribute(
+    selected_keys: list[K], owed: dict[K, int], num_rollouts: int
+) -> list[int]:
+    """Spread each key's owed rollouts over its selection instances, in order —
+    content-identical tasks are interchangeable, so any instance can absorb the
+    debt (capped at `num_rollouts` each). Returns one count per selection."""
+    remaining = dict(owed)
+    counts: list[int] = []
+    for key in selected_keys:
+        take = min(num_rollouts, remaining.get(key, 0))
+        if take:
+            remaining[key] -= take
+        counts.append(take)
+    return counts
+
 
 def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
     """Pull `--resume <dir>` / `--resume=<dir>` out of argv, returning (dir, the other args).
@@ -51,25 +82,27 @@ def load_resume_config(resume_dir: Path) -> EvalConfig:
 
 def load(
     resume_dir: Path,
-    selected_idxs: list[int],
+    selected_keys: list[K],
     num_rollouts: int,
     complete: Callable[[Episode], bool] | None = None,
     *,
     whole_task: bool = False,
-) -> tuple[list[Episode], dict[int, int]]:
-    """Load the good saved rollouts as finished episodes and diff them against the
-    run's target: returns (kept episodes, rollouts owed per task idx). A rollout is
-    kept or redone as a unit — the episode — so a multi-trace rollout interrupted
-    mid-write is simply owed again. `complete` is the environment's keep-verdict
-    (`Env.complete`); without it (the server path) the default is
-    `episode.ok`, so an errored rollout is dropped and re-run. `whole_task` redoes
-    a partially-kept task as a unit — the legacy group-scored path, where
-    `run_group` always serves the full n. Rewrites `traces.jsonl` to just the kept
-    rows via a temp file + atomic rename, so an interrupted resume can't corrupt
-    the prior results; the resumed rollouts then append. Pre-episode files (one
-    bare trace per line) load each trace as a single-trace episode."""
+    key_of: Callable[[Mapping], K] | None = None,
+) -> tuple[list[Episode], dict[K, int]]:
+    """Load the good saved rollouts and diff them against the run's target: returns
+    (kept episodes, rollouts owed per task key). `selected_keys` is one key per
+    selected task (duplicates allowed — a key selected k times is owed up to
+    `k * num_rollouts`; spread back over the tasks with `distribute`). `key_of` maps
+    a saved row's task data to its key (default `task_key`; the legacy bridge uses
+    row indices). `complete` is the keep-verdict (default `episode.ok`); `whole_task`
+    redoes a partially-kept task whole (legacy group scoring). Rewrites
+    `traces.jsonl` to the kept rows via a temp file + atomic rename; a torn or
+    malformed row is owed again, never a crash."""
     path = resume_dir / TRACES_FILE
-    selected = set(selected_idxs)
+    targets = {
+        key: count * num_rollouts for key, count in Counter(selected_keys).items()
+    }
+    keyed = key_of if key_of is not None else task_key
 
     def parse(row: dict) -> Episode:
         if sniff_episode(row):
@@ -77,7 +110,7 @@ def load(
         return Episode.of(WireTrace.model_validate(row))
 
     verdict = complete if complete is not None else (lambda episode: episode.ok)
-    good: dict[int, list[tuple[bytes, Episode]]] = defaultdict(list)
+    good: dict[K, list[tuple[bytes, Episode]]] = defaultdict(list)
     if path.exists():
         with path.open("rb") as results:
             for line in results:
@@ -89,16 +122,16 @@ def load(
                     except ValueError:
                         row = json.loads(line)
                     # The task rides each trace; a traceless record (a failure
-                    # before any trace minted) has no idx and is owed again.
+                    # before any trace minted) has no task and is owed again.
                     if sniff_episode(row):
-                        idx = row["traces"][0]["task"]["data"]["idx"]
+                        key = keyed(row["traces"][0]["task"]["data"])
                     else:
-                        idx = row["task"]["data"]["idx"]
+                        key = keyed(row["task"]["data"])
                 except (ValueError, KeyError, IndexError, TypeError):
                     # A torn final line (the run died mid-write) or a foreign shape
                     # is not a keepable rollout — it's owed again, never a crash.
                     continue
-                if idx not in selected or len(good[idx]) >= num_rollouts:
+                if key not in targets or len(good[key]) >= targets[key]:
                     continue
                 try:
                     episode = parse(row)
@@ -106,20 +139,20 @@ def load(
                         continue
                 except Exception:  # malformed row: redo it
                     continue
-                good[idx].append(
+                good[key].append(
                     (line if line.endswith(b"\n") else line + b"\n", episode)
                 )
     keep: list[bytes] = []
     episodes: list[Episode] = []
-    owed: dict[int, int] = {}
-    for idx in selected_idxs:
-        rows = good.get(idx, [])
-        if whole_task and len(rows) < num_rollouts:
+    owed: dict[K, int] = {}
+    for key, target in targets.items():
+        rows = good.get(key, [])
+        if whole_task and len(rows) < target:
             rows = []  # a partial unit redoes whole — its kept rows are dropped
         keep.extend(line for line, _ in rows)
         episodes.extend(episode for _, episode in rows)
-        if missing := num_rollouts - len(rows):
-            owed[idx] = missing
+        if missing := target - len(rows):
+            owed[key] = missing
     tmp = path.with_suffix(".jsonl.tmp")
     tmp.write_bytes(b"".join(keep))
     tmp.replace(path)

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/cli/eval/runner.py

@@ -32,21 +32,25 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
     )
     out = output_path(config)
-    owed: dict[str, int] | None = None
+    # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts.
+    plan = [(task, config.num_rollouts) for task in tasks]
     # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run.
     finished: list[Episode] = []
     if config.resume is not None:
-        finished, owed = resume.load(
-            out, [t.data.idx for t in tasks], config.num_rollouts, env.complete
-        )
+        keys = [
+            resume.task_key(t.data.model_dump(mode="json", exclude_none=True))
+            for t in tasks
+        ]
+        finished, owed = resume.load(out, keys, config.num_rollouts, env.complete)
         if not owed:  # already complete - report it and exit successfully
             print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts))
             raise SystemExit(0)
-        tasks = [task for task in tasks if owed.get(task.data.idx)]
+        counts = resume.distribute(keys, owed, config.num_rollouts)
+        plan = [(task, n) for task, n in zip(tasks, counts) if n]
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
             out,
-            len(tasks),
+            len(plan),
             sum(owed.values()),
         )
     else:
@@ -70,13 +74,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     # Serving resources (shared tool servers, interception) come up once for the
     # run; plan slots inside so the env's agents borrow them.
     async with env.serving():
-        planned = [
-            slot
-            for task in tasks
-            for slot in env.slots(
-                task, n=owed[task.data.idx] if owed else config.num_rollouts
-            )
-        ]
+        planned = [slot for task, n in plan for slot in env.slots(task, n=n)]
         slots = [RunSlot.finished(episode) for episode in finished] + planned
         push_state = None
         if config.push and config.rich:
@@ -123,6 +121,15 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         if legacy
         else {"config_data": env_config_data(config.env)}  # picklable across the spawn
     )
+    tasks = []
+    if not legacy:
+        from verifiers.v1.loaders import load_taskset
+
+        # The client owns the taskset: load it here, once — the server (and its pool
+        # workers) never load data, they rebuild each dispatched task from its request.
+        tasks = load_taskset(config.env.taskset).select(
+            config.num_tasks, config.shuffle
+        )
     # Spawned processes inherit no logging — hand them the main process's setup so
     # their rollout logs land in the output dir.
     level = "DEBUG" if config.verbose else "INFO"
@@ -151,48 +158,57 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         address = await asyncio.to_thread(address_queue.get, timeout=600)
         client = EnvClient(address=address)
         await client.wait_for_server_startup(timeout=600)
-        info = await client.info()
-        # Only a legacy (v0) env group-scores; a v1 env scores siblings in its own
-        # rollout.
-        group_scored = info.requires_group_scoring
-        if info.num_tasks is None:  # infinite taskset - the run must be bounded
-            if config.num_tasks is None:
-                raise ValueError(
-                    f"{config.env_id} is infinite - bound the run with -n/--num-tasks"
-                )
-            if config.shuffle:
-                logger.warning(
-                    "shuffle is a no-op on an infinite taskset - "
-                    "taking the first %d generated tasks",
-                    config.num_tasks,
-                )
-            idxs = list(range(config.num_tasks))
-        else:
+        # A v1 run dispatches — and resumes — tasks by content: the client owns them,
+        # and `resume.task_key` is their identity. Only the legacy bridge is addressed
+        # by dataset row (its dataset lives server-side, reported via `info`), and
+        # only a legacy env group-scores; a v1 env scores siblings in its own rollout.
+        if legacy:
+            info = await client.info()
+            group_scored = info.requires_group_scoring
             idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
+            plan = [({"task_idx": idx}, config.num_rollouts) for idx in idxs]
+        else:
+            group_scored = False
+            plan = [
+                ({"task_data": task.data.model_dump(mode="json")}, config.num_rollouts)
+                for task in tasks
+            ]
         out = output_path(config)
         finished: list[Episode] = []
         if config.resume is not None:
-            # (legacy only) a group is served and scored together, so a partially-kept
-            # task redoes as a whole group — whole_task drops its kept rows.
-            finished, owed = resume.load(
-                out, idxs, config.num_rollouts, whole_task=group_scored
-            )
+            if legacy:
+                # A group is served and scored together, so a partially-kept task
+                # redoes as a whole group — whole_task drops its kept rows.
+                finished, owed = resume.load(
+                    out,
+                    idxs,
+                    config.num_rollouts,
+                    whole_task=group_scored,
+                    key_of=lambda data: data.get("idx"),
+                )
+                counts = resume.distribute(idxs, owed, config.num_rollouts)
+            else:
+                keys = [
+                    resume.task_key(t.data.model_dump(mode="json", exclude_none=True))
+                    for t in tasks
+                ]
+                finished, owed = resume.load(out, keys, config.num_rollouts)
+                counts = resume.distribute(keys, owed, config.num_rollouts)
             if not owed:  # already complete - report it and exit successfully
-                print(resume.nothing_to_resume_msg(out, len(idxs), config.num_rollouts))
+                print(resume.nothing_to_resume_msg(out, len(plan), config.num_rollouts))
                 raise SystemExit(0)
-            idxs = [idx for idx in idxs if owed.get(idx)]
+            plan = [(payload, n) for (payload, _), n in zip(plan, counts) if n]
             logger.info(
                 "resuming %s: %d task(s), %d rollout(s) owed",
                 out,
-                len(idxs),
+                len(plan),
                 sum(owed.values()),
             )
         else:
-            owed = {idx: config.num_rollouts for idx in idxs}
             save_config(config, out)
             logger.info(
                 "running %dx%d rollouts via the env-server %s pool on %s",
-                len(idxs),
+                len(plan),
                 config.num_rollouts,
                 config.pool.type,
                 config.model,
@@ -224,13 +240,13 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 records.append(Episode.of(trace))
             return records
 
-        async def run_unit(idx: int) -> list[Episode]:
+        async def run_unit(payload: dict) -> list[Episode]:
             async with semaphore or contextlib.nullcontext():
                 episode = await client.run(
-                    task_idx=idx,
                     client=config.client,
                     model=config.model,
                     sampling=config.sampling,
+                    **payload,
                 )
             for trace in episode.traces:
                 trace.stamp(EvalRunInfo(id=config.uuid))
@@ -238,12 +254,12 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
             return [episode]
 
         # A group-scored legacy task runs its rollouts together (one `run_group`
-        # request, one worker); otherwise each rollout is its own `run_rollout`
-        # request, dispatched least-busy across workers.
+        # request, one worker); otherwise each rollout is its own `run` request,
+        # dispatched least-busy across workers.
         units = (
-            [run_group_unit(i) for i in idxs]
+            [run_group_unit(payload["task_idx"]) for payload, _ in plan]
             if group_scored
-            else [run_unit(i) for i in idxs for _ in range(owed[i])]
+            else [run_unit(payload) for payload, n in plan for _ in range(n)]
         )
         results = await asyncio.gather(*units)
         await client.close()

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/cli/gepa.py

@@ -66,7 +66,7 @@ def main(argv: list[str] | None = None) -> None:
     # Refuse multi-agent before the dry-run return, so --dry-run can't write a
     # config the real invocation would reject.
     env_cls = vf.environment_class(
-        config.env.taskset.id if config.env.taskset is not None else "",
+        config.env.taskset.id,
         config.env.id,
     )
     if not issubclass(env_cls, vf.SingleAgentEnv):

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/cli/output.py

@@ -36,8 +36,8 @@ def output_path(config: EvalConfig) -> Path:
     if config.output_dir is not None:
         return config.output_dir
     taskset = config.env.taskset
-    env = taskset.name if taskset is not None else "no-taskset"
-    if taskset is not None and taskset.id and config.env.id:
+    env = taskset.name if taskset.id else "no-taskset"
+    if taskset.id and config.env.id:
         # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k-v1` run must
         # not share a parent dir with a plain `gsm8k-v1` one.
         env = f"{env_name(config.env.id)}+{env}"

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/configs/env.py

@@ -138,9 +138,7 @@ class EnvServerConfig(BaseConfig):
     @property
     def is_legacy(self) -> bool:
         """A v0/legacy env (run via the bridge): a legacy `id` is set and no v1 taskset."""
-        return self.id is not None and (
-            self.env.taskset is None or not self.env.taskset.id
-        )
+        return self.id is not None and not self.env.taskset.id
 
     @property
     def env_id(self) -> str:
@@ -152,7 +150,7 @@ class EnvServerConfig(BaseConfig):
     def _refuse_legacy_id_with_taskset(self):
         """A legacy `id` next to a v1 `env.taskset` would be silently inert
         (`is_legacy` is False and the v0 env never loads); refuse the mix."""
-        if self.id is not None and self.env.taskset is not None and self.env.taskset.id:
+        if self.id is not None and self.env.taskset.id:
             raise ValueError(
                 f"--id {self.id!r} is the legacy (v0) env id and can't combine with "
                 f"the v1 taskset {self.env.taskset.id!r}. Pairing an env with a "

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/env.py

@@ -31,7 +31,7 @@ from verifiers.v1.retries import RetryConfig, run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.errors import EnvError, boundary
 from verifiers.v1.task import Task, resolve_server_config
-from verifiers.v1.taskset import Taskset, TasksetConfig
+from verifiers.v1.taskset import TasksetConfig
 from verifiers.v1.episode import Episode
 from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.generic import deep_merge, generic_type
@@ -65,9 +65,9 @@ class EnvConfig(BaseConfig):
     """Which `Env` runs. Empty = the taskset's own, else `SingleAgentEnv`; set
     to pair a reusable env with any taskset (an explicit id wins over the bundled)."""
     # SerializeAsAny: the env-server wire needs the resolved subclass's fields.
-    taskset: SerializeAsAny[TasksetConfig] | None = None
+    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
     """The seed taskset — the rows every rollout starts from (`--env.taskset.id`).
-    None only for an env that mints its tasks without a dataset."""
+    The id stays empty only for a legacy (v0) run, which sets the top-level `id`."""
     timeout: TimeoutConfig = TimeoutConfig()
     retries: RetryConfig = RetryConfig()
     """Whole-EPISODE retries — the coarse fallback for faults no agent owns; a
@@ -81,17 +81,14 @@ class EnvConfig(BaseConfig):
     @property
     def env_id(self) -> str:
         """The taskset id, prefixed by the paired env id (`best-of-n+gsm8k-v1`)."""
-        taskset_id = self.taskset.id if self.taskset is not None else ""
-        if taskset_id and self.id:
-            return f"{self.id}+{taskset_id}"
-        return taskset_id or self.id
+        if self.taskset.id and self.id:
+            return f"{self.id}+{self.taskset.id}"
+        return self.taskset.id or self.id
 
     def agent_harnesses(self) -> dict[str, HarnessConfig]:
         """Each declared role's resolved harness config (pin, else the taskset's
         default) — known without constructing the env."""
-        default = default_agent_harness(
-            self.taskset.id if self.taskset is not None else ""
-        )
+        default = default_agent_harness(self.taskset.id)
         return {
             name: cfg.harness if cfg.harness is not None else default
             for name, cfg in _declared_agent_configs(self).items()
@@ -239,7 +236,7 @@ class Env(ABC, Generic[ConfigT]):
                 f"{config_cls.__name__}(...) explicitly"
             )
         self.config: ConfigT = config
-        if config.taskset is None:
+        if not config.taskset.id:
             raise ValueError(
                 f"{type(self).__name__} needs a seed taskset — every rollout starts "
                 "from one of its tasks: set --env.taskset.id (or the positional "
@@ -247,7 +244,7 @@ class Env(ABC, Generic[ConfigT]):
             )
         self.taskset = load_taskset(config.taskset)
         self._default_harness = default_agent_harness(config.taskset.id)
-        task_cls = generic_type(type(self.taskset), Task, origin=Taskset) or Task
+        task_cls = type(self.taskset).task_type()
         self._task_cls: type[Task] = task_cls
         self._agent_specs: dict[str, AgentConfig] = _declared_agent_configs(self.config)
         if not self._agent_specs:
@@ -520,7 +517,7 @@ class Env(ABC, Generic[ConfigT]):
         """`requires_tunnel` over the consumers known before any rollout: role
         runtimes, live `shared` servers, and the task class's tool/user servers;
         a class overriding `server_config` conservatively counts as remote."""
-        task_cls = generic_type(type(self.taskset), Task, origin=Taskset) or Task
+        task_cls = type(self.taskset).task_type()
         server_classes = [*task_cls.tools, *([task_cls.user] if task_cls.user else [])]
         if server_classes and task_cls.server_config is not Task.server_config:
             return True

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/harness.py

@@ -4,6 +4,7 @@ import logging
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
+from pathlib import Path
 from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 
 from pydantic import Field
@@ -41,6 +42,10 @@ class HarnessConfig(BaseConfig):
     forward_env: list[str] = Field(default_factory=list)
     """Host variables to forward without writing secrets into config; explicit `env` wins."""
     disabled_tools: list[str] | None = None
+    skills: list[Path] = Field(default_factory=list)
+    """Skill folders to upload into the program's skill discovery directory — each
+    lands at `<skills dir>/<folder name>`. Only harnesses whose program discovers
+    skills natively (`SUPPORTS_SKILLS`) accept them."""
 
     @property
     def name(self) -> str:
@@ -66,6 +71,15 @@ class Harness(ABC, Generic[ConfigT]):
     every real harness; the tool-less chat loops (`null`) override to False. Read
     where model-directed execution changes the rules: the subprocess-on-host
     warning, the judge env's sandbox requirement."""
+    SUPPORTS_SKILLS: ClassVar[bool] = False
+    """Whether the program discovers SKILL.md skills — its `setup` calls
+    `install_skills` with the program's fixed discovery location; configuring
+    `skills` on a harness without support is rejected up front."""
+    NEEDS_CONTAINER: ClassVar[bool] = True
+    """Whether the program must run in a container runtime: True for every harness
+    that installs and drives a third-party program — on the host (subprocess) it
+    leaks host state (auth, config, processes) both ways. Only the minimal
+    in-house loops (`bash`, `null`) override to False."""
 
     def __init__(self, config: ConfigT) -> None:
         self.config = config
@@ -102,6 +116,28 @@ class Harness(ABC, Generic[ConfigT]):
     async def setup(self, runtime: Runtime) -> None:
         """Provision this harness in `runtime` before its execution timeout starts."""
 
+    async def install_skills(self, runtime: Runtime, dest: str) -> None:
+        """Upload each `config.skills` folder into `runtime` at `dest/<folder name>` —
+        the program's fixed skill discovery location, which a supporting harness's
+        `setup` passes."""
+        for skill in self.config.skills:
+            # Resolve so `.`/`..` entries get their real folder name (and can't
+            # place files outside `dest`).
+            skill = skill.resolve()
+            if not skill.is_dir():
+                raise ValueError(f"skill {str(skill)!r} is not a folder")
+            executables = []
+            for file in sorted(skill.rglob("*")):
+                if not file.is_file():
+                    continue
+                target = f"{dest}/{skill.name}/{file.relative_to(skill).as_posix()}"
+                await runtime.write(target, file.read_bytes())
+                if os.access(file, os.X_OK):
+                    executables.append(target)
+            if executables:
+                # `write` moves bytes, not modes; restore the execute bits scripts need.
+                await runtime.run(["chmod", "+x", *executables], {})
+
     async def run(
         self,
         ctx: ModelContext,

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/bash/harness.py

@@ -41,6 +41,7 @@ class BashHarness(Harness[BashHarnessConfig]):
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True
     SUPPORTS_MESSAGE_PROMPT = True
+    NEEDS_CONTAINER = False
 
     async def setup(self, runtime: Runtime) -> None:
         await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/claude_code/harness.py

@@ -13,6 +13,7 @@ from verifiers.v1.trace import Trace
 CLAUDE_HOME = "/tmp/vf-claude-code-{version}"
 CLAUDE_BIN = f"{CLAUDE_HOME}/.local/bin/claude"
 CLAUDE_CONFIG_DIR = ".vf-claude"
+SKILLS_DIR = f"{CLAUDE_CONFIG_DIR}/skills"
 INSTALL = """
 set -e
 command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
@@ -30,8 +31,10 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
     SUPPORTS_MCP = True
     # images would require streaming inputs
     SUPPORTS_MESSAGE_PROMPT = False
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         home = CLAUDE_HOME.format(version=self.config.version)
         binary = CLAUDE_BIN.format(version=self.config.version)
         script = INSTALL.format(version=self.config.version, home=home)
@@ -66,13 +69,13 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
             "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
             "ANTHROPIC_API_KEY": secret,
             "CLAUDE_CONFIG_DIR": CLAUDE_CONFIG_DIR,
+            "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
             "DISABLE_AUTOUPDATER": "1",
             "IS_SANDBOX": "1",
         }
         argv = [
             CLAUDE_BIN.format(version=self.config.version),
             "--print",
-            "--bare",
             "--dangerously-skip-permissions",
             "--no-session-persistence",
             "--model",

{verifiers-0.2.2.dev16 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/codex/harness.py

@@ -23,6 +23,7 @@ KEY_VAR = "CODEX_INTERCEPT_KEY"
 
 CODEX_DIR = "/tmp/vf-codex"
 CODEX_BIN = f"{CODEX_DIR}/bin/codex"
+SKILLS_DIR = ".agents/skills"
 INSTALL = r"""
 set -e
 mkdir -p {dir}/bin
@@ -46,8 +47,10 @@ class CodexHarness(Harness[CodexHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = False  # TODO
     SUPPORTS_MCP = False  # TODO
     SUPPORTS_MESSAGE_PROMPT = True
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         logger.info("codex: ensuring codex %s is installed", self.config.version)
         script = (
             INSTALL.replace("{version}", self.config.version)