verifiers 0.2.1.dev19__tar.gz → 0.2.1.dev21__tar.gz

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev19
+Version: 0.2.1.dev21
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/tests/v1/test_color_codeword_tasks.py

@@ -10,9 +10,9 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
         return render(color, size)
 
     monkeypatch.setattr(taskset, "color_data_url", counted)
-    tasks = taskset.ColorCodewordTaskset(
-        taskset.ColorCodewordConfig(num_examples=100)
-    ).load()
+    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
+        num_tasks=100
+    )
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_MAP)

verifiers-0.2.1.dev21/tests/v1/test_taskset.py

@@ -0,0 +1,71 @@
+"""`Taskset.select` over list, generator, and `INFINITE` `load` implementations."""
+
+import itertools
+
+import pytest
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        for i in range(10):
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+def idxs(tasks: list[CountTask]) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def test_select_bounds_an_infinite_taskset() -> None:
+    tasks = InfiniteTaskset(vf.TasksetConfig()).select(num_tasks=5)
+    assert idxs(tasks) == [0, 1, 2, 3, 4]
+
+
+def test_select_infinite_requires_num_tasks() -> None:
+    with pytest.raises(ValueError, match="infinite"):
+        InfiniteTaskset(vf.TasksetConfig()).select()
+
+
+def test_select_infinite_ignores_shuffle() -> None:
+    tasks = InfiniteTaskset(vf.TasksetConfig()).select(num_tasks=3, shuffle=True)
+    assert idxs(tasks) == [0, 1, 2]
+
+
+def test_select_finite() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    assert len(taskset.select()) == 10
+    assert idxs(taskset.select(num_tasks=4)) == [0, 1, 2, 3]
+
+
+def test_select_shuffle_samples_the_whole_taskset_reproducibly() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    first = idxs(taskset.select(num_tasks=5, shuffle=True))
+    assert first == idxs(taskset.select(num_tasks=5, shuffle=True))
+    assert len(first) == 5 and set(first) <= set(range(10))
+    assert first != [0, 1, 2, 3, 4]  # sampled from the whole set, not the head
+
+
+def test_select_only_builds_what_the_run_takes() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+    RecordingTaskset(vf.TasksetConfig()).select(num_tasks=3)
+    assert built == [0, 1, 2]

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/cli/dashboard/eval.py

@@ -40,6 +40,7 @@ _LABEL_WIDTH = len("timeouts")
 
 _STYLE = {
     "pending": "dim",
+    "boot": "orange3",
     "setup": "yellow",
     "running": "cyan",
     "finalize": "magenta",
@@ -49,6 +50,7 @@ _STYLE = {
 }
 _MARK_LABEL = {
     "pending": "pending",
+    "boot": "boot",
     "setup": "setup",
     "running": "rollout",
     "finalize": "finalize",
@@ -317,7 +319,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
             if judge.cost is not None:
                 total_judge_cost += judge.cost
             have_judge = True
-        for phase in ("setup", "generation", "finalize", "scoring"):
+        for phase in ("boot", "setup", "generation", "finalize", "scoring"):
             span = getattr(trace.timing, phase)
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
@@ -351,7 +353,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
         grid.add_row("usage", "  ·  ".join(usage))
     time_segments = [
         f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"
-        for phase in ("setup", "generation", "finalize", "scoring")
+        for phase in ("boot", "setup", "generation", "finalize", "scoring")
         if phase_count.get(phase)
     ]
     if time_segments:
@@ -383,11 +385,12 @@ def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
 
 
 def _started(rollout: Rollout) -> float:
-    # Sort key: when a rollout began (its setup start). A still-pending rollout has no trace
-    # yet, so it sorts last (+inf) — behind everything already in flight, in task order.
-    return (
-        rollout.trace.timing.setup.start if rollout.trace is not None else float("inf")
-    )
+    # Sort key: when a rollout began (its boot start; setup for pre-boot-span traces on
+    # resume). A still-pending rollout has no trace yet, so it sorts last (+inf) — behind
+    # everything already in flight, in task order.
+    if rollout.trace is None:
+        return float("inf")
+    return rollout.trace.timing.boot.start or rollout.trace.timing.setup.start
 
 
 def _groups(rollouts: list[Rollout]) -> list[list[Rollout]]:
@@ -450,14 +453,14 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
             )
             runtime = f"{runtime_type}({descriptor})" if descriptor else runtime_type
             turns = t.num_turns
-            start = t.timing.setup.start
+            start = t.timing.boot.start or t.timing.setup.start
             end = (
                 t.timing.scoring.end
                 or t.timing.finalize.end
                 or t.timing.generation.end
-                # a rollout that errored in setup has only setup.end — freeze there once done,
-                # else (still running) the timer would grow off `now` forever
-                or (t.timing.setup.end if t.is_completed else 0)
+                # a rollout that errored in boot/setup has only that span's end — freeze there
+                # once done, else (still running) the timer would grow off `now` forever
+                or ((t.timing.setup.end or t.timing.boot.end) if t.is_completed else 0)
                 or now
             )
             prompt, completion, cached, reasoning, nbranches = _tokens(t)

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/cli/debug.py

@@ -30,7 +30,6 @@ from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace, TraceTask
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
-from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -138,15 +137,19 @@ def record_debug_error(
     setup_timeout: float | None,
     action_timeout: float | None,
 ) -> None:
-    if not trace.timing.setup.end:
-        trace.timing.setup.end = time.time()
-    if trace.timing.generation.start and not trace.timing.generation.end:
-        trace.timing.generation.end = time.time()
+    now = time.time()
+    for span in (trace.timing.boot, trace.timing.setup, trace.timing.generation):
+        if span.start and not span.end:
+            span.end = now
     in_action = bool(trace.timing.generation.start)
-    stage = "debug action" if in_action else "setup"
+    stage = (
+        "debug action" if in_action else "setup" if trace.timing.setup.start else "boot"
+    )
     timeout = action_timeout if in_action else setup_timeout
     error_start = (
-        trace.timing.generation.start if in_action else trace.timing.setup.start
+        trace.timing.generation.start
+        if in_action
+        else trace.timing.setup.start or trace.timing.boot.start
     )
     debug.update(error_info(error, error_start, timeout, stage))
     debug.setdefault("runtime", runtime_info(runtime))
@@ -226,8 +229,11 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         else task.data.timeout.setup
     )
     try:
-        trace.timing.setup.start = time.time()
+        trace.timing.boot.start = time.time()
         await runtime.start()
+        now = time.time()
+        trace.timing.boot.end = now
+        trace.timing.setup.start = now
         debug["runtime"] = runtime_info(runtime)
         await asyncio.wait_for(
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
@@ -267,7 +273,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
+    tasks = taskset.select(config.num_tasks, config.shuffle)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/cli/eval/runner.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     client = resolve_client(config.client)
-    tasks = sample(env.taskset.load(), config.shuffle, config.num_tasks)
+    tasks = env.taskset.select(config.num_tasks, config.shuffle)
     ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     # One episode of `num_rollouts` rollouts per task; the shared semaphore bounds total
     # concurrent rollouts (across episodes), so group rewards still see their whole episode.
@@ -162,7 +162,20 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
         await client.wait_for_server_startup(timeout=600)
         info = await client.info()
         group_scored = info.requires_group_scoring
-        idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
+        if info.num_tasks is None:  # infinite taskset - the run must be bounded
+            if config.num_tasks is None:
+                raise ValueError(
+                    f"{config.env_id} is infinite - bound the run with -n/--num-tasks"
+                )
+            if config.shuffle:
+                logger.warning(
+                    "shuffle is a no-op on an infinite taskset - "
+                    "taking the first %d generated tasks",
+                    config.num_tasks,
+                )
+            idxs = list(range(config.num_tasks))
+        else:
+            idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
         out = output_path(config)
         finished: list[Trace] = []
         if config.resume is not None:

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/cli/validate.py

@@ -26,7 +26,6 @@ from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
-from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -198,7 +197,7 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
+    tasks = taskset.select(config.num_tasks, config.shuffle)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/configs/debug.py

@@ -28,6 +28,7 @@ class DebugConfig(BaseConfig):
     for the debug command/script."""
     num_tasks: int | None = Field(
         None,
+        ge=1,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
     )
     """How many tasks to debug (None = all)."""

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/configs/eval.py

@@ -22,6 +22,7 @@ class EvalConfig(EnvServerConfig):
     sampling: SamplingConfig = SamplingConfig()
     num_tasks: int | None = Field(
         None,
+        ge=1,
         validation_alias=AliasChoices("batch_size", "num_examples", "num_tasks", "n"),
     )
     """How many tasks to evaluate (None = all)."""

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/configs/validate.py

@@ -26,6 +26,7 @@ class ValidateConfig(BaseConfig):
     """Run only `Task.setup` and `Task.validate`."""
     num_tasks: int | None = Field(
         None,
+        ge=1,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
     )
     """How many tasks to validate (None = all)."""

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/gepa/config.py

@@ -42,7 +42,7 @@ class GEPAConfig(EnvConfig):
     """Tasks held out to score each candidate system prompt for the pareto frontier."""
     shuffle: bool = Field(True, validation_alias=AliasChoices("shuffle", "s"))
     """Shuffle tasks before splitting into train/val — v1 tasksets have no generic train/val
-    split, so GEPA carves one out of `taskset.load()` the way `run_eval` samples (fixed
+    split, so GEPA carves one out of `Taskset.select` the way `run_eval` samples (fixed
     seed, so the split is reproducible across runs)."""
     seed: int = 0
     """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/gepa/dataset.py

@@ -1,14 +1,13 @@
 """Train/val split and upfront validation for a GEPA run.
 
 v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
-individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of
-`taskset.load()` itself: the shared fixed-seed shuffle (`verifiers.v1.utils.sampling.sample`,
-reproducible across runs like every other entrypoint), then two disjoint slices.
+individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of the tasks
+`Taskset.select` hands it (the shared fixed-seed shuffle, reproducible across runs like every
+other entrypoint): two disjoint slices.
 """
 
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.task import Task
-from verifiers.v1.utils.sampling import sample
 
 
 def reject_group_reward_tasksets(tasks: list[Task]) -> None:
@@ -26,19 +25,16 @@ def reject_group_reward_tasksets(tasks: list[Task]) -> None:
 
 
 def split_tasks(
-    tasks: list[Task], num_train: int, num_val: int, shuffle: bool
+    tasks: list[Task], num_train: int, num_val: int
 ) -> tuple[list[Task], list[Task]]:
-    """The taskset's tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
+    """The selected tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
     minibatches; `val` scores each candidate for the pareto frontier)."""
-    pool = sample(
-        tasks, shuffle
-    )  # shared fixed-seed shuffle; GEPA does its own slicing below
-    if num_train + num_val > len(pool):
+    if num_train + num_val > len(tasks):
         raise ValueError(
             f"requested {num_train} train + {num_val} val tasks, but the taskset only "
-            f"loaded {len(pool)}"
+            f"loaded {len(tasks)}"
         )
-    return pool[:num_train], pool[num_train : num_train + num_val]
+    return tasks[:num_train], tasks[num_train : num_train + num_val]
 
 
 def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> str:

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/gepa/runner.py

@@ -38,10 +38,8 @@ class _GEPALog:
 
 def run_gepa(env: Environment, config: GEPAConfig) -> GEPAResult:
     logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
-    all_tasks = env.taskset.load()
-    train_tasks, val_tasks = split_tasks(
-        all_tasks, config.num_train, config.num_val, config.shuffle
-    )
+    all_tasks = env.taskset.select(config.num_train + config.num_val, config.shuffle)
+    train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val)
     selected_tasks = [*train_tasks, *val_tasks]
     reject_group_reward_tasksets(
         selected_tasks

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/legacy.py

@@ -329,7 +329,7 @@ class LegacyEnvServer(EnvServer):
             self.dataset = self.env.get_dataset()
         except ValueError:
             self.dataset = self.env.get_eval_dataset()
-        self.tasks = self.dataset  # `len(self.tasks)` drives the `info` response
+        self.num_tasks: int | None = len(self.dataset)  # drives the `info` response
         self.requires_group_scoring = self.env.requires_group_rollouts
         self._clients: dict[tuple[str, str], Any] = {}
 

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/rollout.py

@@ -37,6 +37,7 @@ logger = logging.getLogger(__name__)
 
 class Phase(StrEnum):
     PENDING = "pending"
+    BOOT = "boot"
     SETUP = "setup"
     RUNNING = "running"
     FINALIZE = "finalize"
@@ -108,8 +109,8 @@ class Rollout:
             state=state_cls(type(self.task))(),
         )
         self.trace = trace  # expose for the --rich dashboard
-        self.phase = Phase.SETUP  # leaving the queue: provisioning starts now
-        trace.timing.setup.start = time.time()
+        self.phase = Phase.BOOT  # leaving the queue: the runtime boots now
+        trace.timing.boot.start = time.time()
         self.runtime = make_runtime(self.runtime_config, name=trace.id)
         runtime = self.runtime
         trace.runtime = runtime.info
@@ -125,6 +126,10 @@ class Rollout:
         try:
             session = RolloutSession(ctx, trace, stops, self.limits)
             await runtime.start()
+            now = time.time()
+            trace.timing.boot.end = now
+            trace.timing.setup.start = now
+            self.phase = Phase.SETUP
             # Task setup and harness provisioning share one setup-stage deadline.
             setup_deadline = (
                 None
@@ -228,6 +233,7 @@ class Rollout:
             trace.is_completed = True
             now = time.time()
             for span in (
+                trace.timing.boot,
                 trace.timing.setup,
                 trace.timing.generation,
                 trace.timing.finalize,

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/serve/server.py

@@ -26,6 +26,9 @@ from verifiers.v1.types import SamplingConfig
 
 logger = logging.getLogger(__name__)
 
+MAX_LAZY_TASKS = 1_000_000
+"""Most tasks an infinite taskset's generator is willing to build (and cache) per worker."""
+
 
 class EnvServer:
     def __init__(
@@ -34,11 +37,20 @@ class EnvServer:
         self.address = address
         self.taskset_id = config.taskset.id
         self.env = Environment(config)
-        self.tasks = self.env.taskset.load()
+        # A finite taskset is materialized up front (its count is served via `info`); an
+        # infinite one is pulled off its generator on demand (see `_task`), so
+        # `num_tasks=None` on the wire ⟺ the taskset is infinite.
+        self._task_iter = iter(self.env.taskset.load())
+        self._tasks: list = []
+        self.num_tasks: int | None = None
+        if not type(self.env.taskset).INFINITE:
+            self._tasks = list(self._task_iter)
+            self.num_tasks = len(self._tasks)
         # One task type per taskset (the authoring contract; its `load()` constructs it),
         # so group scoring is a run-wide property.
-        self.requires_group_scoring = bool(self.tasks) and bool(
-            discover_decorated(self.tasks[0], "group_reward")
+        first = self._task(0) if self.num_tasks != 0 else None
+        self.requires_group_scoring = first is not None and bool(
+            discover_decorated(first, "group_reward")
         )
         self._clients: dict[
             tuple[str, str], Client
@@ -73,6 +85,26 @@ class EnvServer:
             # of a spurious multiprocessing traceback, matching serve_env's own handling.
             pass
 
+    def _task(self, idx: int):
+        """The task at `idx`; an infinite taskset is generated (and cached) up to `idx`
+        on demand. Generation must be deterministic — every pool worker runs its own
+        `load()`, so idx-addressing relies on all of them producing the same sequence.
+        Lazy generation is capped at `MAX_LAZY_TASKS`: an idx that far ahead is a
+        runaway driver, and generating (and caching) toward it would hang the worker
+        and exhaust memory instead of failing the one request."""
+        while len(self._tasks) <= idx:
+            if idx >= MAX_LAZY_TASKS:
+                raise IndexError(
+                    f"task_idx {idx} exceeds the lazy-generation cap ({MAX_LAZY_TASKS})"
+                )
+            try:
+                self._tasks.append(next(self._task_iter))
+            except StopIteration:
+                raise IndexError(
+                    f"task_idx {idx} out of range ({len(self._tasks)} tasks)"
+                ) from None
+        return self._tasks[idx]
+
     def _client(self, client_config: ClientConfig, model: str) -> Client:
         """Cache clients because renderer initialization builds a tokenizer pool."""
         key = (client_config.model_dump_json(), model)
@@ -96,14 +128,14 @@ class EnvServer:
 
     async def _run_rollout(self, req: RunRolloutRequest) -> RunRolloutResponse:
         ctx = self._context(req.client, req.model, req.sampling)
-        episode = self.env.episode(self.tasks[req.task_idx], ctx, n=1)
+        episode = self.env.episode(self._task(req.task_idx), ctx, n=1)
         traces = await episode.run()
         # Trust the concrete trace; serialize it once before client-side re-typing.
         return RunRolloutResponse.model_construct(trace=traces[0])
 
     async def _run_group(self, req: RunGroupRequest) -> RunGroupResponse:
         ctx = self._context(req.client, req.model, req.sampling)
-        episode = self.env.episode(self.tasks[req.task_idx], ctx, n=req.n)
+        episode = self.env.episode(self._task(req.task_idx), ctx, n=req.n)
         traces = await episode.run()
         # Avoid a dump-and-validate copy for every trusted trace in the group.
         return RunGroupResponse.model_construct(traces=traces)
@@ -118,7 +150,7 @@ class EnvServer:
                 response: BaseResponse = HealthResponse()
             elif route == "info":
                 response = InfoResponse(
-                    num_tasks=len(self.tasks),
+                    num_tasks=self.num_tasks,
                     requires_group_scoring=self.requires_group_scoring,
                 )
             elif route == "run_rollout":
@@ -151,10 +183,10 @@ class EnvServer:
 
     async def run(self) -> None:
         logger.info(
-            "EnvServer up: taskset=%s address=%s tasks=%d group_scoring=%s",
+            "EnvServer up: taskset=%s address=%s tasks=%s group_scoring=%s",
             self.taskset_id,
             self.address,
-            len(self.tasks),
+            self.num_tasks if self.num_tasks is not None else "infinite",
             self.requires_group_scoring,
         )
         poller = zmq.asyncio.Poller()

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/serve/types.py

@@ -1,6 +1,6 @@
 from typing import ClassVar
 
-from pydantic import BaseModel, field_serializer
+from pydantic import BaseModel, Field, field_serializer
 
 from verifiers.v1.clients.config import ClientConfig
 from verifiers.v1.task import WireTaskData
@@ -32,14 +32,15 @@ class InfoRequest(BaseRequest):
 
 
 class InfoResponse(BaseResponse):
-    num_tasks: int = 0
+    num_tasks: int | None = None
+    """Task count; `None` means the taskset is infinite (bound runs with `num_tasks`)."""
     requires_group_scoring: bool = False
     """Whether tasks must be run and resumed as whole groups."""
 
 
 class RunRolloutRequest(BaseRequest):
     method: ClassVar[str] = "run_rollout"
-    task_idx: int
+    task_idx: int = Field(ge=0)
     client: ClientConfig
     model: str
     sampling: SamplingConfig
@@ -56,7 +57,7 @@ class RunRolloutResponse(BaseResponse):
 
 class RunGroupRequest(BaseRequest):
     method: ClassVar[str] = "run_group"
-    task_idx: int
+    task_idx: int = Field(ge=0)
     n: int
     client: ClientConfig
     model: str

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev21}/verifiers/v1/taskset.py

@@ -1,12 +1,17 @@
 """The taskset: a thin loader that yields typed tasks.
 
 A `Taskset` is the data half of an environment: config in, tasks out. `load()` — the one
-subclass hook, and the one method consumers call — builds each row's `TaskData` and wraps
-it in the task type with the config's task-facing subtree:
+subclass hook — builds each row's `TaskData` and wraps it in the task type with the
+config's task-facing subtree:
 
-    def load(self) -> list[MyTask]:
+    def load(self) -> Iterable[MyTask]:
         return [MyTask(MyData(idx=i, ...), self.config.task) for i in ...]
 
+`load` may also be a generator — yielding each task as it's built, possibly forever (a
+procedural taskset; declare `INFINITE = True`). Runs materialize the tasks they need
+through `select`, which pulls only that many off a generator; the env server instead
+pulls `load` on demand, task by task.
+
 Load-time knobs (dataset, split, seed) live on the taskset config; the task-facing knobs
 under its `task` subtree (`TasksetConfig.task`, a `TaskConfig`, everything under
 `--taskset.task.*`); a worker-scoped shared tool server is declared on `tools` with its knobs
@@ -22,6 +27,9 @@ replay can rebuild every saved row as the declared type's data and re-wrap it.
 
 from __future__ import annotations
 
+import itertools
+import logging
+from collections.abc import Iterable
 from typing import TYPE_CHECKING, ClassVar, Generic
 
 from pydantic import SerializeAsAny
@@ -31,10 +39,13 @@ from typing_extensions import TypeVar
 from verifiers.v1.task import TaskConfig, TaskT, resolve_server_config
 from verifiers.v1.types import ID
 from verifiers.v1.utils.install import env_name
+from verifiers.v1.utils.sampling import sample
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Toolset
 
+logger = logging.getLogger(__name__)
+
 
 class TasksetConfig(BaseConfig):
     id: ID = ""
@@ -51,15 +62,46 @@ TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetC
 
 
 class Taskset(Generic[TaskT, TasksetConfigT]):
+    INFINITE: ClassVar[bool] = False
+    """Whether `load` yields tasks forever (a procedural generator). Infinity is inherent
+    to the taskset, not a config knob: consumers bound a run with `select(num_tasks)`
+    (`-n` on the CLI), and shuffle is impossible."""
+
     tools: ClassVar[tuple[type[Toolset], ...]] = ()
     """Tool server classes shared by one environment worker's rollouts."""
 
     def __init__(self, config: TasksetConfigT) -> None:
         self.config = config
 
-    def load(self) -> list[TaskT]:
+    def load(self) -> Iterable[TaskT]:
         raise NotImplementedError
 
+    def select(
+        self, num_tasks: int | None = None, shuffle: bool = False
+    ) -> list[TaskT]:
+        """Materialize the tasks a run needs: the first `num_tasks` off `load` (all of
+        them when `None`), pulled lazily — a generator `load` only builds what the run
+        takes. `shuffle` samples the subset from the whole taskset instead (the shared
+        fixed-seed shuffle, `verifiers.v1.utils.sampling`), which materializes everything
+        first; on an `INFINITE` taskset it's a no-op (warned) — the first `num_tasks`
+        generated tasks are already an arbitrary sample."""
+        if type(self).INFINITE:
+            if num_tasks is None:
+                raise ValueError(
+                    f"{type(self).__name__} is infinite - select a bounded subset "
+                    "with num_tasks (-n on the CLI)"
+                )
+            if shuffle:
+                logger.warning(
+                    "shuffle is a no-op on an infinite taskset - "
+                    "taking the first %d generated tasks",
+                    num_tasks,
+                )
+            return list(itertools.islice(self.load(), num_tasks))
+        if shuffle:
+            return sample(self.load(), shuffle=True, limit=num_tasks)
+        return list(itertools.islice(self.load(), num_tasks))
+
     def server_config(self, server_cls: type) -> BaseConfig:
         """The config a `tools` entry is built with, resolved off `self.config` (the
         taskset config; see `resolve_server_config`). Override to pair explicitly."""