verifiers 0.2.1.dev20__tar.gz → 0.2.1.dev21__tar.gz

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev20
+Version: 0.2.1.dev21
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/tests/v1/test_color_codeword_tasks.py

@@ -10,9 +10,9 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
         return render(color, size)
 
     monkeypatch.setattr(taskset, "color_data_url", counted)
-    tasks = taskset.ColorCodewordTaskset(
-        taskset.ColorCodewordConfig(num_examples=100)
-    ).load()
+    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
+        num_tasks=100
+    )
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_MAP)

verifiers-0.2.1.dev21/tests/v1/test_taskset.py

@@ -0,0 +1,71 @@
+"""`Taskset.select` over list, generator, and `INFINITE` `load` implementations."""
+
+import itertools
+
+import pytest
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        for i in range(10):
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+def idxs(tasks: list[CountTask]) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def test_select_bounds_an_infinite_taskset() -> None:
+    tasks = InfiniteTaskset(vf.TasksetConfig()).select(num_tasks=5)
+    assert idxs(tasks) == [0, 1, 2, 3, 4]
+
+
+def test_select_infinite_requires_num_tasks() -> None:
+    with pytest.raises(ValueError, match="infinite"):
+        InfiniteTaskset(vf.TasksetConfig()).select()
+
+
+def test_select_infinite_ignores_shuffle() -> None:
+    tasks = InfiniteTaskset(vf.TasksetConfig()).select(num_tasks=3, shuffle=True)
+    assert idxs(tasks) == [0, 1, 2]
+
+
+def test_select_finite() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    assert len(taskset.select()) == 10
+    assert idxs(taskset.select(num_tasks=4)) == [0, 1, 2, 3]
+
+
+def test_select_shuffle_samples_the_whole_taskset_reproducibly() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    first = idxs(taskset.select(num_tasks=5, shuffle=True))
+    assert first == idxs(taskset.select(num_tasks=5, shuffle=True))
+    assert len(first) == 5 and set(first) <= set(range(10))
+    assert first != [0, 1, 2, 3, 4]  # sampled from the whole set, not the head
+
+
+def test_select_only_builds_what_the_run_takes() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+    RecordingTaskset(vf.TasksetConfig()).select(num_tasks=3)
+    assert built == [0, 1, 2]

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/cli/debug.py

@@ -30,7 +30,6 @@ from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace, TraceTask
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
-from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -274,7 +273,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
+    tasks = taskset.select(config.num_tasks, config.shuffle)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/cli/eval/runner.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     client = resolve_client(config.client)
-    tasks = sample(env.taskset.load(), config.shuffle, config.num_tasks)
+    tasks = env.taskset.select(config.num_tasks, config.shuffle)
     ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     # One episode of `num_rollouts` rollouts per task; the shared semaphore bounds total
     # concurrent rollouts (across episodes), so group rewards still see their whole episode.
@@ -162,7 +162,20 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
         await client.wait_for_server_startup(timeout=600)
         info = await client.info()
         group_scored = info.requires_group_scoring
-        idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
+        if info.num_tasks is None:  # infinite taskset - the run must be bounded
+            if config.num_tasks is None:
+                raise ValueError(
+                    f"{config.env_id} is infinite - bound the run with -n/--num-tasks"
+                )
+            if config.shuffle:
+                logger.warning(
+                    "shuffle is a no-op on an infinite taskset - "
+                    "taking the first %d generated tasks",
+                    config.num_tasks,
+                )
+            idxs = list(range(config.num_tasks))
+        else:
+            idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
         out = output_path(config)
         finished: list[Trace] = []
         if config.resume is not None:

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/cli/validate.py

@@ -26,7 +26,6 @@ from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
-from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -198,7 +197,7 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
+    tasks = taskset.select(config.num_tasks, config.shuffle)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/configs/debug.py

@@ -28,6 +28,7 @@ class DebugConfig(BaseConfig):
     for the debug command/script."""
     num_tasks: int | None = Field(
         None,
+        ge=1,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
     )
     """How many tasks to debug (None = all)."""

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/configs/eval.py

@@ -22,6 +22,7 @@ class EvalConfig(EnvServerConfig):
     sampling: SamplingConfig = SamplingConfig()
     num_tasks: int | None = Field(
         None,
+        ge=1,
         validation_alias=AliasChoices("batch_size", "num_examples", "num_tasks", "n"),
     )
     """How many tasks to evaluate (None = all)."""

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/configs/validate.py

@@ -26,6 +26,7 @@ class ValidateConfig(BaseConfig):
     """Run only `Task.setup` and `Task.validate`."""
     num_tasks: int | None = Field(
         None,
+        ge=1,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
     )
     """How many tasks to validate (None = all)."""

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/gepa/config.py

@@ -42,7 +42,7 @@ class GEPAConfig(EnvConfig):
     """Tasks held out to score each candidate system prompt for the pareto frontier."""
     shuffle: bool = Field(True, validation_alias=AliasChoices("shuffle", "s"))
     """Shuffle tasks before splitting into train/val — v1 tasksets have no generic train/val
-    split, so GEPA carves one out of `taskset.load()` the way `run_eval` samples (fixed
+    split, so GEPA carves one out of `Taskset.select` the way `run_eval` samples (fixed
     seed, so the split is reproducible across runs)."""
     seed: int = 0
     """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/gepa/dataset.py

@@ -1,14 +1,13 @@
 """Train/val split and upfront validation for a GEPA run.
 
 v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
-individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of
-`taskset.load()` itself: the shared fixed-seed shuffle (`verifiers.v1.utils.sampling.sample`,
-reproducible across runs like every other entrypoint), then two disjoint slices.
+individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of the tasks
+`Taskset.select` hands it (the shared fixed-seed shuffle, reproducible across runs like every
+other entrypoint): two disjoint slices.
 """
 
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.task import Task
-from verifiers.v1.utils.sampling import sample
 
 
 def reject_group_reward_tasksets(tasks: list[Task]) -> None:
@@ -26,19 +25,16 @@ def reject_group_reward_tasksets(tasks: list[Task]) -> None:
 
 
 def split_tasks(
-    tasks: list[Task], num_train: int, num_val: int, shuffle: bool
+    tasks: list[Task], num_train: int, num_val: int
 ) -> tuple[list[Task], list[Task]]:
-    """The taskset's tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
+    """The selected tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
     minibatches; `val` scores each candidate for the pareto frontier)."""
-    pool = sample(
-        tasks, shuffle
-    )  # shared fixed-seed shuffle; GEPA does its own slicing below
-    if num_train + num_val > len(pool):
+    if num_train + num_val > len(tasks):
         raise ValueError(
             f"requested {num_train} train + {num_val} val tasks, but the taskset only "
-            f"loaded {len(pool)}"
+            f"loaded {len(tasks)}"
         )
-    return pool[:num_train], pool[num_train : num_train + num_val]
+    return tasks[:num_train], tasks[num_train : num_train + num_val]
 
 
 def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> str:

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/gepa/runner.py

@@ -38,10 +38,8 @@ class _GEPALog:
 
 def run_gepa(env: Environment, config: GEPAConfig) -> GEPAResult:
     logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
-    all_tasks = env.taskset.load()
-    train_tasks, val_tasks = split_tasks(
-        all_tasks, config.num_train, config.num_val, config.shuffle
-    )
+    all_tasks = env.taskset.select(config.num_train + config.num_val, config.shuffle)
+    train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val)
     selected_tasks = [*train_tasks, *val_tasks]
     reject_group_reward_tasksets(
         selected_tasks

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/legacy.py

@@ -329,7 +329,7 @@ class LegacyEnvServer(EnvServer):
             self.dataset = self.env.get_dataset()
         except ValueError:
             self.dataset = self.env.get_eval_dataset()
-        self.tasks = self.dataset  # `len(self.tasks)` drives the `info` response
+        self.num_tasks: int | None = len(self.dataset)  # drives the `info` response
         self.requires_group_scoring = self.env.requires_group_rollouts
         self._clients: dict[tuple[str, str], Any] = {}
 

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/serve/server.py

@@ -26,6 +26,9 @@ from verifiers.v1.types import SamplingConfig
 
 logger = logging.getLogger(__name__)
 
+MAX_LAZY_TASKS = 1_000_000
+"""Most tasks an infinite taskset's generator is willing to build (and cache) per worker."""
+
 
 class EnvServer:
     def __init__(
@@ -34,11 +37,20 @@ class EnvServer:
         self.address = address
         self.taskset_id = config.taskset.id
         self.env = Environment(config)
-        self.tasks = self.env.taskset.load()
+        # A finite taskset is materialized up front (its count is served via `info`); an
+        # infinite one is pulled off its generator on demand (see `_task`), so
+        # `num_tasks=None` on the wire ⟺ the taskset is infinite.
+        self._task_iter = iter(self.env.taskset.load())
+        self._tasks: list = []
+        self.num_tasks: int | None = None
+        if not type(self.env.taskset).INFINITE:
+            self._tasks = list(self._task_iter)
+            self.num_tasks = len(self._tasks)
         # One task type per taskset (the authoring contract; its `load()` constructs it),
         # so group scoring is a run-wide property.
-        self.requires_group_scoring = bool(self.tasks) and bool(
-            discover_decorated(self.tasks[0], "group_reward")
+        first = self._task(0) if self.num_tasks != 0 else None
+        self.requires_group_scoring = first is not None and bool(
+            discover_decorated(first, "group_reward")
         )
         self._clients: dict[
             tuple[str, str], Client
@@ -73,6 +85,26 @@ class EnvServer:
             # of a spurious multiprocessing traceback, matching serve_env's own handling.
             pass
 
+    def _task(self, idx: int):
+        """The task at `idx`; an infinite taskset is generated (and cached) up to `idx`
+        on demand. Generation must be deterministic — every pool worker runs its own
+        `load()`, so idx-addressing relies on all of them producing the same sequence.
+        Lazy generation is capped at `MAX_LAZY_TASKS`: an idx that far ahead is a
+        runaway driver, and generating (and caching) toward it would hang the worker
+        and exhaust memory instead of failing the one request."""
+        while len(self._tasks) <= idx:
+            if idx >= MAX_LAZY_TASKS:
+                raise IndexError(
+                    f"task_idx {idx} exceeds the lazy-generation cap ({MAX_LAZY_TASKS})"
+                )
+            try:
+                self._tasks.append(next(self._task_iter))
+            except StopIteration:
+                raise IndexError(
+                    f"task_idx {idx} out of range ({len(self._tasks)} tasks)"
+                ) from None
+        return self._tasks[idx]
+
     def _client(self, client_config: ClientConfig, model: str) -> Client:
         """Cache clients because renderer initialization builds a tokenizer pool."""
         key = (client_config.model_dump_json(), model)
@@ -96,14 +128,14 @@ class EnvServer:
 
     async def _run_rollout(self, req: RunRolloutRequest) -> RunRolloutResponse:
         ctx = self._context(req.client, req.model, req.sampling)
-        episode = self.env.episode(self.tasks[req.task_idx], ctx, n=1)
+        episode = self.env.episode(self._task(req.task_idx), ctx, n=1)
         traces = await episode.run()
         # Trust the concrete trace; serialize it once before client-side re-typing.
         return RunRolloutResponse.model_construct(trace=traces[0])
 
     async def _run_group(self, req: RunGroupRequest) -> RunGroupResponse:
         ctx = self._context(req.client, req.model, req.sampling)
-        episode = self.env.episode(self.tasks[req.task_idx], ctx, n=req.n)
+        episode = self.env.episode(self._task(req.task_idx), ctx, n=req.n)
         traces = await episode.run()
         # Avoid a dump-and-validate copy for every trusted trace in the group.
         return RunGroupResponse.model_construct(traces=traces)
@@ -118,7 +150,7 @@ class EnvServer:
                 response: BaseResponse = HealthResponse()
             elif route == "info":
                 response = InfoResponse(
-                    num_tasks=len(self.tasks),
+                    num_tasks=self.num_tasks,
                     requires_group_scoring=self.requires_group_scoring,
                 )
             elif route == "run_rollout":
@@ -151,10 +183,10 @@ class EnvServer:
 
     async def run(self) -> None:
         logger.info(
-            "EnvServer up: taskset=%s address=%s tasks=%d group_scoring=%s",
+            "EnvServer up: taskset=%s address=%s tasks=%s group_scoring=%s",
             self.taskset_id,
             self.address,
-            len(self.tasks),
+            self.num_tasks if self.num_tasks is not None else "infinite",
             self.requires_group_scoring,
         )
         poller = zmq.asyncio.Poller()

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/serve/types.py

@@ -1,6 +1,6 @@
 from typing import ClassVar
 
-from pydantic import BaseModel, field_serializer
+from pydantic import BaseModel, Field, field_serializer
 
 from verifiers.v1.clients.config import ClientConfig
 from verifiers.v1.task import WireTaskData
@@ -32,14 +32,15 @@ class InfoRequest(BaseRequest):
 
 
 class InfoResponse(BaseResponse):
-    num_tasks: int = 0
+    num_tasks: int | None = None
+    """Task count; `None` means the taskset is infinite (bound runs with `num_tasks`)."""
     requires_group_scoring: bool = False
     """Whether tasks must be run and resumed as whole groups."""
 
 
 class RunRolloutRequest(BaseRequest):
     method: ClassVar[str] = "run_rollout"
-    task_idx: int
+    task_idx: int = Field(ge=0)
     client: ClientConfig
     model: str
     sampling: SamplingConfig
@@ -56,7 +57,7 @@ class RunRolloutResponse(BaseResponse):
 
 class RunGroupRequest(BaseRequest):
     method: ClassVar[str] = "run_group"
-    task_idx: int
+    task_idx: int = Field(ge=0)
     n: int
     client: ClientConfig
     model: str

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/taskset.py

@@ -1,12 +1,17 @@
 """The taskset: a thin loader that yields typed tasks.
 
 A `Taskset` is the data half of an environment: config in, tasks out. `load()` — the one
-subclass hook, and the one method consumers call — builds each row's `TaskData` and wraps
-it in the task type with the config's task-facing subtree:
+subclass hook — builds each row's `TaskData` and wraps it in the task type with the
+config's task-facing subtree:
 
-    def load(self) -> list[MyTask]:
+    def load(self) -> Iterable[MyTask]:
         return [MyTask(MyData(idx=i, ...), self.config.task) for i in ...]
 
+`load` may also be a generator — yielding each task as it's built, possibly forever (a
+procedural taskset; declare `INFINITE = True`). Runs materialize the tasks they need
+through `select`, which pulls only that many off a generator; the env server instead
+pulls `load` on demand, task by task.
+
 Load-time knobs (dataset, split, seed) live on the taskset config; the task-facing knobs
 under its `task` subtree (`TasksetConfig.task`, a `TaskConfig`, everything under
 `--taskset.task.*`); a worker-scoped shared tool server is declared on `tools` with its knobs
@@ -22,6 +27,9 @@ replay can rebuild every saved row as the declared type's data and re-wrap it.
 
 from __future__ import annotations
 
+import itertools
+import logging
+from collections.abc import Iterable
 from typing import TYPE_CHECKING, ClassVar, Generic
 
 from pydantic import SerializeAsAny
@@ -31,10 +39,13 @@ from typing_extensions import TypeVar
 from verifiers.v1.task import TaskConfig, TaskT, resolve_server_config
 from verifiers.v1.types import ID
 from verifiers.v1.utils.install import env_name
+from verifiers.v1.utils.sampling import sample
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Toolset
 
+logger = logging.getLogger(__name__)
+
 
 class TasksetConfig(BaseConfig):
     id: ID = ""
@@ -51,15 +62,46 @@ TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetC
 
 
 class Taskset(Generic[TaskT, TasksetConfigT]):
+    INFINITE: ClassVar[bool] = False
+    """Whether `load` yields tasks forever (a procedural generator). Infinity is inherent
+    to the taskset, not a config knob: consumers bound a run with `select(num_tasks)`
+    (`-n` on the CLI), and shuffle is impossible."""
+
     tools: ClassVar[tuple[type[Toolset], ...]] = ()
     """Tool server classes shared by one environment worker's rollouts."""
 
     def __init__(self, config: TasksetConfigT) -> None:
         self.config = config
 
-    def load(self) -> list[TaskT]:
+    def load(self) -> Iterable[TaskT]:
         raise NotImplementedError
 
+    def select(
+        self, num_tasks: int | None = None, shuffle: bool = False
+    ) -> list[TaskT]:
+        """Materialize the tasks a run needs: the first `num_tasks` off `load` (all of
+        them when `None`), pulled lazily — a generator `load` only builds what the run
+        takes. `shuffle` samples the subset from the whole taskset instead (the shared
+        fixed-seed shuffle, `verifiers.v1.utils.sampling`), which materializes everything
+        first; on an `INFINITE` taskset it's a no-op (warned) — the first `num_tasks`
+        generated tasks are already an arbitrary sample."""
+        if type(self).INFINITE:
+            if num_tasks is None:
+                raise ValueError(
+                    f"{type(self).__name__} is infinite - select a bounded subset "
+                    "with num_tasks (-n on the CLI)"
+                )
+            if shuffle:
+                logger.warning(
+                    "shuffle is a no-op on an infinite taskset - "
+                    "taking the first %d generated tasks",
+                    num_tasks,
+                )
+            return list(itertools.islice(self.load(), num_tasks))
+        if shuffle:
+            return sample(self.load(), shuffle=True, limit=num_tasks)
+        return list(itertools.islice(self.load(), num_tasks))
+
     def server_config(self, server_cls: type) -> BaseConfig:
         """The config a `tools` entry is built with, resolved off `self.config` (the
         taskset config; see `resolve_server_config`). Override to pair explicitly."""

{verifiers-0.2.1.dev20 → verifiers-0.2.1.dev21}/verifiers/v1/tasksets/textarena/taskset.py

@@ -6,8 +6,10 @@ outcome there; scoring reads that file instead of trusting conversational text.
 """
 
 import copy
+import itertools
 import json
 import random
+from collections.abc import Iterator
 from typing import Literal
 
 import verifiers.v1 as vf
@@ -88,7 +90,6 @@ class TextArenaConfig(vf.TasksetConfig):
         "WordLadder-v0",
         "WordSearch-v0",
     ]
-    num_tasks: int = 1000
     task: TextArenaTaskConfig = TextArenaTaskConfig()
 
 
@@ -115,7 +116,9 @@ class TextArenaTask(vf.Task[TextArenaData, TextArenaState, TextArenaTaskConfig])
 
 
 class TextArenaTaskset(vf.Taskset[TextArenaTask, TextArenaConfig]):
-    def load(self) -> list[TextArenaTask]:
+    INFINITE = True
+
+    def load(self) -> Iterator[TextArenaTask]:
         nltk.download("words", quiet=True)
         nltk.download("averaged_perceptron_tagger_eng", quiet=True)
         template = ta.make(env_id=self.config.game)
@@ -130,8 +133,8 @@ class TextArenaTaskset(vf.Taskset[TextArenaTask, TextArenaConfig]):
         # seed-invariant; otherwise each task must expose its seeded board.
         first = observation(0)
         seed_specific = observation(1) != first
-        return [
-            TextArenaTask(
+        for i in itertools.count():
+            yield TextArenaTask(
                 TextArenaData(
                     idx=i,
                     name=f"{self.config.game}#{i}",
@@ -141,8 +144,6 @@ class TextArenaTaskset(vf.Taskset[TextArenaTask, TextArenaConfig]):
                 ),
                 self.config.task,
             )
-            for i in range(self.config.num_tasks)
-        ]
 
 
 if __name__ == "__main__":

verifiers-0.2.1.dev21/verifiers/v1/utils/sampling.py

@@ -0,0 +1,25 @@
+"""Shared sampling: an optional fixed-seed shuffle, then an optional head-slice.
+
+Every entrypoint narrows its items the same way: with `--shuffle`, a shuffle under a fixed
+seed so the sampled subset is the *same* every run (reproducible), then an optional slice to
+the first `limit`. Taskset entrypoints (eval, validate, debug, GEPA) go through
+`Taskset.select`, which shares this shuffle; the server eval path and the legacy bridge
+sample plain index lists here directly.
+"""
+
+import random
+from collections.abc import Iterable
+from typing import TypeVar
+
+SEED = 0  # fixed so `--shuffle` samples the same items every run (reproducible)
+
+T = TypeVar("T")
+
+
+def sample(items: Iterable[T], shuffle: bool, limit: int | None = None) -> list[T]:
+    """`items` optionally shuffled under the fixed seed, then optionally sliced to the first
+    `limit`. Returns a new list; the input is left untouched."""
+    items = list(items)
+    if shuffle:
+        random.Random(SEED).shuffle(items)
+    return items if limit is None else items[:limit]

verifiers-0.2.1.dev20/verifiers/v1/utils/sampling.py

@@ -1,26 +0,0 @@
-"""Shared task sampling: an optional fixed-seed shuffle, then an optional head-slice.
-
-Every taskset entrypoint — eval, server eval, debug, validate, legacy eval, and GEPA — narrows
-the loaded tasks (or their indices) the same way: with `--shuffle`, a shuffle under a fixed seed
-so the sampled subset is the *same* every run (reproducible), then an optional slice to the
-first `limit`. GEPA layers its disjoint train/val split on top of this shuffle (see
-`verifiers.v1.gepa.dataset`).
-"""
-
-import random
-from typing import TypeVar
-
-_SHUFFLE_SEED = (
-    0  # fixed so `--shuffle` samples the same items every run (reproducible)
-)
-
-T = TypeVar("T")
-
-
-def sample(items: list[T], shuffle: bool, limit: int | None = None) -> list[T]:
-    """`items` optionally shuffled under the fixed seed, then optionally sliced to the first
-    `limit`. Returns a new list; the input is left untouched."""
-    items = list(items)
-    if shuffle:
-        random.Random(_SHUFFLE_SEED).shuffle(items)
-    return items if limit is None else items[:limit]