verifiers 0.3.2.dev151__py3-none-any.whl → 0.3.2.dev152__py3-none-any.whl

verifiers/v1/__init__.py

@@ -23,6 +23,7 @@ from verifiers.v1.configs.env import EnvConfig, SharedEnvConfig, default_agent_h
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig, Judges
 from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.select import SelectCLIConfig, SelectConfig, TaskMatchConfig
 from verifiers.v1.configs.serve import (
     ElasticPoolConfig,
     ServeConfig,
@@ -289,6 +290,9 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     # taskset / harness / runtime / environment
     "Taskset",
     "TaskConfig",
+    "SelectConfig",
+    "SelectCLIConfig",
+    "TaskMatchConfig",
     "TasksetConfig",
     "SharedTasksetConfig",
     "DecoratedFunctionConfig",

verifiers/v1/cli/debug.py

@@ -269,15 +269,12 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
-    taskset = vf.load_taskset(config.taskset)
-    if config.num_tasks is None and taskset.INFINITE:
+    taskset = vf.load_taskset(config.taskset).select(config.select)
+    if not taskset.bounded:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
         )
-    selected = taskset.shuffle() if config.shuffle else taskset
-    if config.num_tasks is not None:
-        selected = selected.head(config.num_tasks)
-    tasks = list(selected)
+    tasks = list(taskset)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

verifiers/v1/cli/eval/runner.py

@@ -82,15 +82,12 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
     from verifiers.v1.utils.loaders import load_environment
 
     env = load_environment(config.env)
-    taskset = env.taskset
-    if config.num_tasks is None and taskset.INFINITE:
+    taskset = env.taskset.select(config.select)
+    if not taskset.bounded:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
         )
-    selected = taskset.shuffle() if config.shuffle else taskset
-    if config.num_tasks is not None:
-        selected = selected.head(config.num_tasks)
-    tasks = list(selected)
+    tasks = list(taskset)
     out = output_path(config)
     # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts.
     plan = [(task, config.num_rollouts) for task in tasks]

verifiers/v1/cli/init.py

@@ -109,7 +109,7 @@ class {prefix}Taskset(vf.Taskset[{prefix}Task, {prefix}Config]):
     def load(self) -> list[{prefix}Task]:
         raise NotImplementedError(
             "Return this taskset's tasks, e.g. "
-            "[{prefix}Task({prefix}Data(idx=i, prompt=...), self.config.task) "
+            "[{prefix}Task({prefix}Data(prompt=...), self.config.task) "
             "for i in range(self.config.num_tasks)]."
         )
 '''

verifiers/v1/cli/validate.py

@@ -325,15 +325,12 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
-    taskset = vf.load_taskset(config.taskset)
-    if config.num_tasks is None and taskset.INFINITE:
+    taskset = vf.load_taskset(config.taskset).select(config.select)
+    if not taskset.bounded:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
         )
-    selected = taskset.shuffle() if config.shuffle else taskset
-    if config.num_tasks is not None:
-        selected = selected.head(config.num_tasks)
-    tasks = list(selected)
+    tasks = list(taskset)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

verifiers/v1/configs/cli/debug.py

@@ -7,6 +7,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.cli.validate import CheckTimeoutConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
@@ -15,6 +16,9 @@ class DebugConfig(BaseConfig):
     uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
     """Auto-generated run id, used as the default output directory leaf."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks to debug, under `--select.*` (`-n` sets
+    `select.limit`, `-s` sets `select.shuffle`)."""
     runtime: RuntimeConfig = PrimeConfig()
     """Where each task's setup hook and debug action run."""
     command: str | None = None
@@ -26,14 +30,6 @@ class DebugConfig(BaseConfig):
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     """Per-task stage timeouts: `--timeout.setup` for the `setup` hook, `--timeout.total`
     for the debug command/script."""
-    num_tasks: int | None = Field(
-        None,
-        ge=1,
-        validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
-    )
-    """How many tasks to debug (None = all)."""
-    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
         128, validation_alias=AliasChoices("max_concurrent", "c")
     )

verifiers/v1/configs/cli/eval.py

@@ -9,6 +9,7 @@ from pydantic_config import BaseConfig
 from verifiers.v1.clients import ClientConfig, EvalClientConfig
 from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
 from verifiers.v1.configs.env import EnvConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
 
@@ -84,12 +85,9 @@ class EvalConfig(BaseConfig):
     """Model id."""
     client: ClientConfig = EvalClientConfig()
     sampling: SamplingConfig = SamplingConfig()
-    num_tasks: int | None = Field(
-        None,
-        ge=1,
-        validation_alias=AliasChoices("batch_size", "num_examples", "num_tasks", "n"),
-    )
-    """How many tasks to evaluate (None = all)."""
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks to evaluate, under `--select.*` (`-n` sets
+    `select.limit`, `-s` sets `select.shuffle`)."""
     num_rollouts: int = Field(
         1,
         ge=1,
@@ -98,8 +96,6 @@ class EvalConfig(BaseConfig):
         ),
     )
     """Independent episodes per task — the trainer's group size."""
-    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
         128, ge=1, validation_alias=AliasChoices("max_concurrent", "c")
     )

verifiers/v1/configs/cli/validate.py

@@ -7,6 +7,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.cli.eval import RunConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
@@ -24,6 +25,9 @@ class ValidateConfig(BaseConfig):
     """Run identity: `run.name` auto-generates as `<taskset>--validate--<short-id>` and
     names the run directory under `output_dir`."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks to validate, under `--select.*` (`-n` sets
+    `select.limit`, `-s` sets `select.shuffle`)."""
     runtime: RuntimeConfig = PrimeConfig()
     """Where each task's validation hooks run."""
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
@@ -31,14 +35,6 @@ class ValidateConfig(BaseConfig):
     """Run only `Task.setup`."""
     only_gold: bool = False
     """Run only `Task.setup` and `Task.validate`."""
-    num_tasks: int | None = Field(
-        None,
-        ge=1,
-        validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
-    )
-    """How many tasks to validate (None = all)."""
-    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
         128, validation_alias=AliasChoices("max_concurrent", "c")
     )

verifiers/v1/configs/select.py

@@ -0,0 +1,118 @@
+"""Which of a taskset's tasks a run uses: `Taskset.select(SelectConfig)`, under
+`--select.*` on the eval, debug, validate and GEPA CLIs (`SelectCLIConfig`)."""
+
+import re
+import sys
+
+from pydantic import AliasChoices, Field, field_validator
+from pydantic_config import BaseConfig
+
+IDX_RANGE = re.compile(r"(\d*):(\d*)(?::(\d*))?")
+
+
+class TaskMatchConfig(BaseConfig):
+    """Tasks named by position or identity; a task matches when any list names it."""
+
+    idx: list[int | str] = Field(default_factory=list)
+    """Positions in the taskset's `load()` stream (`TaskData.idx`): ints and Python
+    slices `start:stop:step`, each part optional (`"100:"`, `":50"`, `"::2"`).
+    Negative positions are not supported. A comma-separated string also works
+    (`"0:10,17"`)."""
+    ids: list[str] = Field(default_factory=list)
+    """`TaskData.id` values."""
+    keys: list[str] = Field(default_factory=list)
+    """`Task.key` values, as recorded on each trace (`task.key`)."""
+    names: list[str] = Field(default_factory=list)
+    """`TaskData.name` values."""
+
+    @field_validator("idx", mode="before")
+    @classmethod
+    def _parse_idx(cls, value):
+        if isinstance(value, (int, str)):
+            value = [value]
+        items: list[int | str] = []
+        for item in value:
+            parts = item.split(",") if isinstance(item, str) else [item]
+            items.extend(_parse_idx_item(part) for part in parts)
+        return items
+
+    @property
+    def empty(self) -> bool:
+        return not (self.idx or self.ids or self.keys or self.names)
+
+    def idx_ranges(self) -> list[range]:
+        """`idx` as ranges; an open-ended slice stops at `sys.maxsize`."""
+        ranges: list[range] = []
+        for item in self.idx:
+            if isinstance(item, int):
+                ranges.append(range(item, item + 1))
+                continue
+            start, stop, step = (item.split(":") + [""])[:3]
+            ranges.append(
+                range(int(start or 0), int(stop or sys.maxsize), int(step or 1))
+            )
+        return ranges
+
+    @property
+    def idx_tail(self) -> bool:
+        """Whether an `idx` slice names every position from some start on."""
+        return any(r.stop == sys.maxsize and r.step == 1 for r in self.idx_ranges())
+
+    @property
+    def idx_stop(self) -> int | None:
+        """One past the last position this can match, when it names only closed `idx`
+        ranges; None when a match could lie anywhere in the stream."""
+        ranges = self.idx_ranges()
+        if not ranges or self.ids or self.keys or self.names:
+            return None
+        stop = max(r.stop for r in ranges)
+        return None if stop == sys.maxsize else stop
+
+
+def _parse_idx_item(item: int | str) -> int | str:
+    if isinstance(item, int):
+        if item < 0:
+            raise ValueError(f"idx {item} is negative")
+        return item
+    text = item.strip()
+    if text.isdigit():
+        return int(text)
+    match = IDX_RANGE.fullmatch(text)
+    if match is None:
+        raise ValueError(f"idx {item!r} is not an int or a 'start:stop:step' slice")
+    start, stop, step = match.groups()
+    if step and int(step) == 0:
+        raise ValueError(f"idx slice {item!r} has step 0")
+    if stop and int(start or 0) >= int(stop):
+        raise ValueError(f"idx range {item!r} is empty")
+    return text
+
+
+class SelectConfig(BaseConfig):
+    """The steps apply in a fixed order: `include` keeps tasks, `exclude` drops them,
+    then `shuffle`, `skip` and `limit` pick from what is left."""
+
+    include: TaskMatchConfig = TaskMatchConfig()
+    """Keep only the tasks this names. Empty keeps all."""
+    exclude: TaskMatchConfig = TaskMatchConfig()
+    """Drop the tasks this names."""
+    shuffle: bool = False
+    """Shuffle the kept tasks under `seed`. Needs a finite stream: an infinite taskset
+    must be bounded by closed `include.idx` ranges first."""
+    seed: int = 0
+    """Seed for `shuffle`, fixed so runs select the same tasks."""
+    skip: int = Field(0, ge=0)
+    """Drop this many tasks after the shuffle."""
+    limit: int | None = Field(None, ge=1)
+    """Take at most this many tasks after `skip` (None = all)."""
+
+
+class SelectCLIConfig(SelectConfig):
+    """`SelectConfig` with CLI short flags, for the one `select` block of an entrypoint:
+    `-n` sets `limit` and `-s` sets `shuffle`."""
+
+    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
+    """Shuffle the kept tasks under `seed` (`-s`). Needs a finite stream: an infinite
+    taskset must be bounded by closed `include.idx` ranges first."""
+    limit: int | None = Field(None, ge=1, validation_alias=AliasChoices("limit", "n"))
+    """Take at most this many tasks after `skip` (`-n`; None = all)."""

verifiers/v1/gepa/config.py

@@ -17,6 +17,7 @@ from verifiers.v1.clients import EvalClientConfig
 from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
 from verifiers.v1.configs.cli.eval import RunConfig, default_run_name
 from verifiers.v1.configs.env import EnvConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
 
@@ -49,17 +50,17 @@ class GEPAConfig(BaseConfig):
     reflection_client: EvalClientConfig | None = None
     """Endpoint for `reflection_model`. None = reuse `client`."""
 
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks GEPA splits into train/val, under `--select.*`
+    (`-n` sets `select.limit`, `-s` sets `select.shuffle`)."""
     num_train: int = Field(100, ge=1)
     """Tasks reserved for reflection minibatches (GEPA never scores the full trainset at once)."""
     num_val: int = Field(50, ge=1)
     """Tasks held out to score each candidate system prompt for the pareto frontier."""
-    shuffle: bool = Field(True, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before splitting into train/val — v1 tasksets have no generic train/val
-    split, so GEPA carves one out of `Taskset.select` the way `run_eval` samples (fixed
-    seed, so the split is reproducible across runs)."""
     seed: int = 0
-    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling
-    uses a fixed seed, matching eval — so this doesn't change the train/val split."""
+    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). The train/val
+    split comes from `select` (`-s` shuffles it under `select.seed`), so this doesn't
+    change it."""
 
     max_total_rollouts: int = Field(500)
     """Total rollouts GEPA may spend across the whole optimization run."""

verifiers/v1/gepa/dataset.py

@@ -2,8 +2,8 @@
 
 v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
 individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of the tasks
-`Taskset.select` hands it (the shared fixed-seed shuffle, reproducible across runs like every
-other entrypoint): two disjoint slices.
+`select` yields (reproducible across runs like every other entrypoint): two
+disjoint slices.
 """
 
 from verifiers.v1.task import Task

verifiers/v1/gepa/runner.py

@@ -38,10 +38,8 @@ class _GEPALog:
 
 def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
     logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
-    # Global shuffle: an infinite taskset raises here — run it with
-    # shuffle=false (there is no whole set to sample from).
-    taskset = env.taskset.shuffle() if config.shuffle else env.taskset
-    all_tasks = list(taskset.head(config.num_train + config.num_val))
+    taskset = env.taskset.select(config.select)
+    all_tasks = list(taskset.take(config.num_train + config.num_val))
     train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val)
     selected_tasks = [*train_tasks, *val_tasks]
     # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool —

verifiers/v1/task.py

@@ -82,7 +82,9 @@ class TaskData(BaseModel):
     model_config = ConfigDict(frozen=True)
 
     idx: int | None = None
-    """Taskset-autogenerated index."""
+    """Task index in the taskset's `load()` stream - automatically set."""
+    id: str | None = None
+    """Optional durable id from the task's source, e.g. a dataset's instance id."""
     name: str | None = None
     """Optional human-readable task name."""
     description: str | None = None
@@ -163,11 +165,15 @@ class Task(Generic[DataT, StateT, ConfigT]):
         """
         return self.hash
 
-    def with_system_prompt(self, system_prompt: str) -> Self:
+    def with_data(self, **update: Any) -> Self:
+        """A shallow copy whose data has `update` applied."""
         clone = copy.copy(self)
-        clone.data = self.data.model_copy(update={"system_prompt": system_prompt})
+        clone.data = self.data.model_copy(update=update)
         return clone
 
+    def with_system_prompt(self, system_prompt: str) -> Self:
+        return self.with_data(system_prompt=system_prompt)
+
     def runtime_env(self) -> dict[str, str]:
         """Live-only process environment; unlike TaskData, it is not traced."""
         return {}

verifiers/v1/taskset.py

@@ -4,25 +4,34 @@ A `Taskset` is the data half of an environment: config in, tasks out. `load()` i
 the main hook that builds each task:
 
     def load(self) -> Iterable[MyTask]:
-        for i in ...:
-            yield MyTask(MyData(idx=i, ...), self.config.task)
+        for row in ...:
+            yield MyTask(MyData(prompt=..., ...), self.config.task)
 
 `load` may also be a generator for infinite tasksets. There is a one-to-one
 mapping between taskset and task type, i.e. a taskset may only yield one task
-type.
+type. Iterating the taskset sets each task's `idx` to its position in `load()`.
+
+Lazy views pick which tasks an iteration yields, chainable in any order:
+
+    taskset.include(idx=["0:100"]).exclude(names=["broken"]).shuffle(seed=0).take(5)
+
+`select(SelectConfig)` chains them in the config's fixed order; the eval, debug,
+validate and GEPA entrypoints and prime-rl all select through it.
 """
 
 from __future__ import annotations
 
 import copy
 import itertools
+import logging
 import random
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterable, Iterator
-from typing import TYPE_CHECKING, Generic, Self
+from typing import TYPE_CHECKING, Any, Generic, Self
 
 from typing_extensions import TypeVar
 
+from verifiers.v1.configs.select import SelectConfig, TaskMatchConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.task import Task, TaskT
 from verifiers.v1.utils.generic import concrete_type
@@ -30,69 +39,138 @@ from verifiers.v1.utils.generic import concrete_type
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Toolset
 
-SEED = 0  # fixed so `--shuffle` samples the same items every run (reproducible)
+logger = logging.getLogger(__name__)
 
 TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetConfig)
 
+Transform = Callable[[Iterator[Any]], Iterator[Any]]
+
 
 class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
     INFINITE: bool = False
-    """Whether the taskset is infinite (yields tasks forever). Class-declared;
-    a `head(n)` view shadows it per instance (bounded by construction)."""
+    """Whether `load()` yields tasks forever. A view can still bound the iteration
+    (see `bounded`)."""
+
+    transform: Transform | None = None
+    """Iteration transform carried by the views (see `_view`)."""
+    _bounded: bool | None = None
+    _ordered: bool = True
+    """Whether the view still yields tasks in `load()` order (no shuffle yet)."""
 
     def __init__(self, config: TasksetConfigT) -> None:
         self.config = config
         override = config.system_prompt
         self.system_prompt = override.read_text() if override is not None else None
-        self.transform: Callable[[Iterator[TaskT]], Iterator[TaskT]] | None = None
-        """Iteration transform carried by `head`/`shuffle` views (see `view`)."""
 
     @abstractmethod
     def load(self) -> Iterable[TaskT]:
         """Build and yield the taskset's tasks; may be a generator (see module doc)."""
 
+    @property
+    def bounded(self) -> bool:
+        """Whether iterating ends: `load()` is finite, or a view caps it (`take`, or
+        `include` with only closed `idx` ranges)."""
+        return not self.INFINITE if self._bounded is None else self._bounded
+
     def __iter__(self) -> Iterator[TaskT]:
-        """Lazily iterate `load()` with the config-layer system prompt applied and
-        any view transform on top — the read path; `load` is the subclass hook."""
-        prompt = self.system_prompt
-        tasks = (
-            task.with_system_prompt(prompt) if prompt is not None else task
-            for task in self.load()
+        """Lazily iterate `load()` with each task's `idx` set to its position and the
+        config-layer system prompt applied, then the views' transform. The views see
+        the final task data, so a `keys` match compares the keys that traces record.
+        This is the read path; `load` is the subclass hook."""
+        update = (
+            {} if self.system_prompt is None else {"system_prompt": self.system_prompt}
         )
-        yield from self.transform(tasks) if self.transform is not None else tasks
-
-    def view(self, transform: Callable[[Iterator[TaskT]], Iterator[TaskT]]) -> Self:
-        """A shallow copy of this taskset iterating through `transform`, composed
-        onto any transform this taskset already carries."""
-        clone = copy.copy(self)
-        prev = self.transform
-        clone.transform = (
-            transform if prev is None else lambda tasks: transform(prev(tasks))
+        tasks: Iterator[TaskT] = (
+            task.with_data(idx=idx, **update) for idx, task in enumerate(self.load())
         )
-        return clone
-
-    def head(self, num_tasks: int) -> Self:
-        """A lazy, always-finite view of the first `num_tasks` tasks."""
-        view = self.view(lambda tasks: itertools.islice(tasks, num_tasks))
-        view.INFINITE = False
+        return tasks if self.transform is None else self.transform(tasks)
+
+    def include(self, **match: Any) -> Self:
+        """A view keeping only the tasks `match` names by `idx`, `ids`, `keys` or
+        `names` (see `TaskMatchConfig`). An identity match on an unbounded view could
+        read forever waiting for a match, so it raises there."""
+        config = TaskMatchConfig(**match)
+        if not self.bounded and (config.ids or config.keys or config.names):
+            raise ValueError(
+                f"{type(self).__name__} is infinite - include by ids, keys or names "
+                "may never end; bound it first with closed include idx ranges"
+            )
+        # Positions arrive in increasing order until a shuffle, so reading can stop
+        # past the last closed idx range.
+        stop = config.idx_stop if self._ordered else None
+        view = self._view(lambda tasks: _match(tasks, config, True, "include", stop))
+        if config.idx_stop is not None:
+            view._bounded = True
         return view
 
-    def shuffle(self, seed: int | None = None) -> Self:
-        """A shuffled view under `seed` — the shared fixed seed when None, so runs
-        sample reproducibly (materializes the receiver on iteration); raises on an
-        infinite taskset — bound it first (`head(n).shuffle()`)."""
-        if self.INFINITE:
+    def exclude(self, **match: Any) -> Self:
+        """A view dropping the tasks `match` names (see `include`). Dropping every
+        position from some start on leaves an unbounded view nothing to yield past
+        it, so that raises there."""
+        config = TaskMatchConfig(**match)
+        if not self.bounded and config.idx_tail:
+            raise ValueError(
+                f"{type(self).__name__} is infinite - excluding an open idx range "
+                "drops every task after its start; exclude a closed range instead"
+            )
+        return self._view(lambda tasks: _match(tasks, config, False, "exclude", None))
+
+    def shuffle(self, seed: int = 0) -> Self:
+        """A shuffled view under `seed` (materializes on iteration); raises on an
+        unbounded view — bound it first with `take` or closed `include` idx ranges.
+        `select` applies `limit` after the shuffle, so there only closed
+        `include.idx` ranges can bound it."""
+        if not self.bounded:
             raise ValueError(
-                f"{type(self).__name__} is infinite - cannot shuffle; "
-                "bound it first with head(num_tasks)"
+                f"{type(self).__name__} is infinite - cannot shuffle; bound it first "
+                "with closed include idx ranges (e.g. --select.include.idx 0:1000), "
+                "or with take(n) before shuffle() in Python"
             )
 
         def shuffled(tasks: Iterator[TaskT]) -> Iterator[TaskT]:
             materialized = list(tasks)
-            random.Random(SEED if seed is None else seed).shuffle(materialized)
+            random.Random(seed).shuffle(materialized)
             return iter(materialized)
 
-        return self.view(shuffled)
+        view = self._view(shuffled)
+        view._ordered = False
+        return view
+
+    def skip(self, num_tasks: int) -> Self:
+        """A view without the first `num_tasks` tasks."""
+        return self._view(lambda tasks: itertools.islice(tasks, num_tasks, None))
+
+    def take(self, num_tasks: int) -> Self:
+        """A lazy, always-finite view of the first `num_tasks` tasks."""
+        view = self._view(lambda tasks: itertools.islice(tasks, num_tasks))
+        view._bounded = True
+        return view
+
+    def select(self, config: SelectConfig) -> Self:
+        """The views `config` describes, chained in its fixed order: `include`,
+        `exclude`, `shuffle`, `skip`, `take`."""
+        view = self
+        if not config.include.empty:
+            view = view.include(**config.include.model_dump())
+        if not config.exclude.empty:
+            view = view.exclude(**config.exclude.model_dump())
+        if config.shuffle:
+            view = view.shuffle(config.seed)
+        if config.skip:
+            view = view.skip(config.skip)
+        if config.limit is not None:
+            view = view.take(config.limit)
+        return view
+
+    def _view(self, transform: Transform) -> Self:
+        """A shallow copy of this taskset iterating through `transform`, composed
+        onto any transform this taskset already carries."""
+        clone = copy.copy(self)
+        prev = self.transform
+        clone.transform = (
+            transform if prev is None else lambda tasks: transform(prev(tasks))
+        )
+        return clone
 
     @classmethod
     def task_type(cls) -> type[Task]:
@@ -109,3 +187,44 @@ class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
                 return [SearchToolset(config.tools)]
         """
         return []
+
+
+def _match(
+    tasks: Iterator[TaskT],
+    match: TaskMatchConfig,
+    keep: bool,
+    label: str,
+    stop: int | None,
+) -> Iterator[TaskT]:
+    """The tasks `match` names (`keep`) or does not name, reading no further than
+    position `stop`. Once reading ends, warns about entries that matched no task, as
+    these are usually typos."""
+    ranges = match.idx_ranges()
+    wanted = {"ids": set(match.ids), "keys": set(match.keys), "names": set(match.names)}
+    found: dict[str, set[str]] = {field: set() for field in wanted}
+    hit_ranges: set[int] = set()
+    for task in tasks:
+        idx = task.data.idx
+        if stop is not None and idx >= stop:
+            break
+        in_ranges = {i for i, r in enumerate(ranges) if idx in r}
+        hit_ranges |= in_ranges
+        values = {
+            "ids": task.data.id,
+            "keys": task.key if match.keys else None,
+            "names": task.data.name,
+        }
+        hits = [f for f, value in values.items() if value in wanted[f]]
+        for field in hits:
+            found[field].add(values[field])
+        if bool(hits or in_ranges) == keep:
+            yield task
+        if stop is not None and idx + 1 >= stop:
+            break
+    missing: dict[str, list] = {
+        f: sorted(wanted[f] - found[f]) for f in wanted if wanted[f] - found[f]
+    }
+    if unmatched := [match.idx[i] for i in range(len(ranges)) if i not in hit_ranges]:
+        missing["idx"] = unmatched
+    if missing:
+        logger.warning("%s matched no task for %s", label, missing)

verifiers/v1/tasksets/nemo_gym/taskset.py

@@ -107,7 +107,6 @@ class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]):
                 request = dialect.parse_request(row["responses_create_params"])
                 yield NeMoGymTask(
                     NeMoGymData(
-                        idx=idx,
                         name=f"{path.stem}:{idx}",
                         prompt=request.messages,
                         row=row,

verifiers/v1/tasksets/openenv/taskset.py

@@ -141,7 +141,6 @@ class OpenEnvTaskset(vf.Taskset[OpenEnvTask, OpenEnvConfig]):
         for idx, reset in enumerate(config.resets):
             yield OpenEnvTask(
                 OpenEnvData(
-                    idx=idx,
                     name=f"{source}#{idx}",
                     prompt=None,
                     env=config.env,

verifiers/v1/tasksets/textarena/taskset.py

@@ -107,7 +107,6 @@ class TextArenaTaskset(vf.Taskset[TextArenaTask, TextArenaConfig]):
         for i in itertools.count():
             yield TextArenaTask(
                 TextArenaData(
-                    idx=i,
                     name=f"{self.config.game}#{i}",
                     prompt=observation(i) if seed_specific else first,
                     system_prompt=SYSTEM_PROMPT,

{verifiers-0.3.2.dev151.dist-info → verifiers-0.3.2.dev152.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev151
+Version: 0.3.2.dev152
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev151.dist-info → verifiers-0.3.2.dev152.dist-info}/RECORD

@@ -1,5 +1,5 @@
 verifiers/__init__.py,sha256=F5IVMw6mhGaPBMCw7QKk0la_r5mMwydP2hdTTdQucik,352
-verifiers/v1/__init__.py,sha256=xbx98CgqJCmGkbR_FP7ugRuDVQ8t_l1F0oVddtLKkSs,8585
+verifiers/v1/__init__.py,sha256=6t2FsosAjmyfJz6qS8tDQ2gWCvQfyRzqk1mYwv5_-EQ,8738
 verifiers/v1/agent.py,sha256=yBCCBe5tVTeQ8OZDlVSZLw_IDvCmEQSnF3wQ1BZcT14,32342
 verifiers/v1/env.py,sha256=GqrVxxrvv-9WitXhD8Rre0tKQgnQWeD8mTM2gDHMVBs,18040
 verifiers/v1/episode.py,sha256=UPtA02aGWA-fY8cJKEuYCMtQVnPN1C0BSN13LggNT8U,5986
@@ -11,21 +11,21 @@ verifiers/v1/rollout.py,sha256=WiMCJyAypsR1OaXvzJQx7G-T67v__AnnDNoXotuI_VE,25444
 verifiers/v1/semantic.py,sha256=VQ_buw6Ya6QtRoh_fiYI5Avs5RmIQCfWH0H0bQc4aoc,4704
 verifiers/v1/session.py,sha256=qVMgKmqkItemR3uYUDAFuDz7huP593kHQbge8FlRba8,28420
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
-verifiers/v1/task.py,sha256=g2MTr1w2TEZXo8g5Z1_SlqqO_mQ9r1VLXrLj4o0Z_VY,12596
-verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
+verifiers/v1/task.py,sha256=6-Tw0iBLsc4hinvVPl_b2EUX8unAAnxtHMXNSIHXsnU,12886
+verifiers/v1/taskset.py,sha256=LE_3ogL7itU2Amk-N20cHkZ7Ev6MznJ0HpC8B1xb4w0,9482
 verifiers/v1/trace.py,sha256=tgzkft4nKF4QGeGXwoqkfXjaUbG65Nn1oBFzMUFp-u0,32234
 verifiers/v1/types.py,sha256=-In5plgB3mQLqtRKHC3QDCPDvuruAqVd8WIjKrsGB_I,10578
 verifiers/v1/acp/__init__.py,sha256=9RySmxFeEMT2XSJy5wYevqEhQdul2jHl9f5XAribG3A,13631
 verifiers/v1/acp/runner.py,sha256=zPo-2ZXmFMmQchhD7nNzlZUrabG09qiBvpB67KVGm3M,14095
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-verifiers/v1/cli/debug.py,sha256=G05-k9mqsMNIf7L-Vlxgdmw3QFwFF94-FodKXV7GYbQ,11822
+verifiers/v1/cli/debug.py,sha256=OgzXCW9HaTNruZil4f70WP5tR6toKliKaVvV98C5qzE,11665
 verifiers/v1/cli/gepa.py,sha256=IyS3YZCCHVydb-OyPeGdtCYsz6OYqXXddXqYq8RLucM,4374
-verifiers/v1/cli/init.py,sha256=6wr14011_Dygv10R6_YWg5A3oSrz187E2WspGRHIfZw,7608
+verifiers/v1/cli/init.py,sha256=k1wawdQBquiiepPrb0enBT-vdKvfSievgs4CiwSdFUc,7601
 verifiers/v1/cli/output.py,sha256=4FRuQ6UIZCigMfUUO2nCFFJKxls3DtWDXswMCVTxjIA,5010
 verifiers/v1/cli/replay.py,sha256=LkeaV8Sxa7kHue9JKHDg0V_N6KXOcjN_r6RmeQVcxfk,10396
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
 verifiers/v1/cli/resume.py,sha256=hqV5AxxqeIInJeEZQerQznuijhMNijFEFcHZIltfLdE,470
-verifiers/v1/cli/validate.py,sha256=Z_rUldl8jzXz52oy3FBIRGr4HYhWFXrq43WmU0sJ-4M,17325
+verifiers/v1/cli/validate.py,sha256=rrEHbGy1RkHt2dnxBqg85VXyB27f3rGGZxlPzZzgmzI,17168
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
 verifiers/v1/cli/dashboard/eval.py,sha256=69vKmLaEY1ll1wahjRGvO1_7QsUQ1URtZOG7ZawvRCw,38347
@@ -35,7 +35,7 @@ verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 verifiers/v1/cli/eval/hint.py,sha256=NXP2_JYrQHguqr68LxOJG3lYAsue_VNoepyTVPn-N4E,256
 verifiers/v1/cli/eval/main.py,sha256=46aHpha6yhFzbqUrRogC5wC2XAwdZcpp-4siqLpUExY,5984
 verifiers/v1/cli/eval/resume.py,sha256=cmFdxLwyU66lyAuXb7U8mNBB-8NLX41JdCg1EuU4w9g,3818
-verifiers/v1/cli/eval/runner.py,sha256=JLitCJajj6h34L0Kq5fWklS7FAsiGOrdGp_c0qVN7Xc,6987
+verifiers/v1/cli/eval/runner.py,sha256=eGnGZUNqt9gG0PK8icesOXbIcIvXuN5157emOM57cd8,6830
 verifiers/v1/clients/__init__.py,sha256=Ysig0tE_0E4Jsfgfes1XHN-fK1s_RCXdqZD6E7lK4PU,507
 verifiers/v1/clients/base.py,sha256=PoDw4GMqrfuPTFVK6n2jDmlD0_lJ5zTrYcNio5yq6K8,1691
 verifiers/v1/clients/client.py,sha256=zqC_AkiD9pl0kxxIbupNiehbS-YnVdp3EMZ0LOHTd8U,3125
@@ -49,16 +49,17 @@ verifiers/v1/configs/harness.py,sha256=ZrR-Egzfft95oMoqQ78bAq9TqP882Q68M_47Ajw9l
 verifiers/v1/configs/judge.py,sha256=WEwA8D4X5JSaIP3c-E2wPnvgByWeT6JRAn1d8gfiQ40,2142
 verifiers/v1/configs/retries.py,sha256=-nPlgmz7J_NZ4GOYG6oCc3thXo4La2SufBmvQGvvPGo,821
 verifiers/v1/configs/runtime.py,sha256=ZEGx5JEgkMOLBQhiKAg6nzqKMhUu4W9tCE3AR--X8HY,5412
+verifiers/v1/configs/select.py,sha256=_HdcepQaJ1sDCYcs7Vqx72npDwHGOgUjsLDZmcgnonU,4743
 verifiers/v1/configs/serve.py,sha256=oa_0BrAVEKboU344w0hYkSaXj8lf26t0H87gvncAo0Q,2716
 verifiers/v1/configs/task.py,sha256=57OnEGAa6jTiG12Lx_elREU02koYh8UkM0t0Vvxahys,2614
 verifiers/v1/configs/taskset.py,sha256=LLbKdV2LW0ZjB9-fsKErIVKjchOAPcDLWGLVjgXKG_0,935
 verifiers/v1/configs/cli/__init__.py,sha256=YMDUPdbRlDJ77dhj3QqzmyRhqNgUvuQdpj_bdBqzZPM,78
-verifiers/v1/configs/cli/debug.py,sha256=6ohG1AB0FBH_s8dVOTre86NlIf_fClz-3FiCBWgR-IM,2854
+verifiers/v1/configs/cli/debug.py,sha256=CcUZ9MEcv0iAL9w2BkTfnZg20yWJ-Vn8P30-e5tuCic,2741
 verifiers/v1/configs/cli/env.py,sha256=FcapEAtgeHzRuVaqk8BU36ZxmMOfdJ1XlioPbdwwspo,3990
-verifiers/v1/configs/cli/eval.py,sha256=1Ptg93IsbNe1IjTp5W8_Xo4gm3Dv2npbYbVXEDRtJfA,7008
+verifiers/v1/configs/cli/eval.py,sha256=9Zw7mAUipMhoIcwRa_lgo5gII4rMmMEBAgzcPxPWPjw,6895
 verifiers/v1/configs/cli/init.py,sha256=JvIUOdhfZfNOXg2wm2i06jLLHpb97tNMzZqmcK_9wVw,916
 verifiers/v1/configs/cli/replay.py,sha256=vUCNs-9j1CXcAZhGozN1EaztcXK_dYVK3iN_fRmbSXw,2929
-verifiers/v1/configs/cli/validate.py,sha256=z7iueq4ndDFKOQ1GA2vaK3C8RhxhDUim-Nn-71T3kGw,3694
+verifiers/v1/configs/cli/validate.py,sha256=oc-SSot6-5a0vXC-yFEQxYbb1O4r8k0Or8d_ijmizkU,3581
 verifiers/v1/dialects/__init__.py,sha256=MNOYM2NdTlSxGK4VtoEjMtD5zya2MpjPDX5jMvZUWEo,736
 verifiers/v1/dialects/anthropic.py,sha256=rkxS8K4xvSTZwS86Mz_a4MfHzrkYf9Vvu8wCerzVNIw,26111
 verifiers/v1/dialects/base.py,sha256=7peaHJGdVT93G3erTnlAcRJi7tGvJ9BUkd_W5O4HGms,16140
@@ -78,10 +79,10 @@ verifiers/v1/envs/user_sim/__init__.py,sha256=nnT695HdqjAEouYL5wtVu6MhIp2wf5yuK-
 verifiers/v1/envs/user_sim/env.py,sha256=deI4r-Utn_v4E_Vb9P_X3LzFh2GRkCO4OlQt8BOCMGY,4942
 verifiers/v1/gepa/__init__.py,sha256=6nmdRE0-34AKioPHBjTxUg5Jo_2z7tMX-OU3zMNpAJI,197
 verifiers/v1/gepa/adapter.py,sha256=YNvHMR2L5Utl-vtPfjZxmDCd2tTcoAPWG6aVGnV1icM,6139
-verifiers/v1/gepa/config.py,sha256=PhZRLkhaPciNm_IhwVX3e9DM8Y2IdCe4Fbi5WjH5aro,4883
-verifiers/v1/gepa/dataset.py,sha256=SPGG55BVbJ9HtCEvm-xyl52QSbldFwGqWvRIcaLelsM,2171
+verifiers/v1/gepa/config.py,sha256=zRjrFkiyMWzD1LXvjuUV_rJtXVO5fWNiraVruHoIADA,4830
+verifiers/v1/gepa/dataset.py,sha256=cS4MtQVoBOnAHKs8REQm5lKGeqUpdXPU1hROU67aBG4,2130
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
-verifiers/v1/gepa/runner.py,sha256=s2eUV0T0mUGy-7XxupoX757z9kZoKAR73NN5M6rUi6E,6096
+verifiers/v1/gepa/runner.py,sha256=-pMy3u5sffEQ4-aDexQviHeQ9NXdNv5rBBnNmhyx_0U,5943
 verifiers/v1/harnesses/__init__.py,sha256=2JPwrRoYoigH-HfBtiFnatRdFEVHLv4rwlKfsFEibyE,1803
 verifiers/v1/harnesses/node.py,sha256=Cci-zCNiVRCtLkJuNOt9iKr7ZIb3ik_YugefXtRyt78,4349
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
@@ -165,12 +166,12 @@ verifiers/v1/tasksets/harbor/toolset.py,sha256=_7HXu-uuvqzhmJ1EX6c3KWLg0g0eRrLjC
 verifiers/v1/tasksets/nemo_gym/__init__.py,sha256=654gAvG3Wd_Z_nh6mypeKn4s_IIQ0Ge6vcZvxTHC2_E,306
 verifiers/v1/tasksets/nemo_gym/response.py,sha256=EhrsquEP8g0bjzAThGfE6Ypgy86_nVX235NiFYtVDVM,4463
 verifiers/v1/tasksets/nemo_gym/server.py,sha256=33C-VDZxPtMV2TFxJVIKijmtBLNlVeRs92ctPE3qAsM,2036
-verifiers/v1/tasksets/nemo_gym/taskset.py,sha256=71h5GGbkbMmUbt4O8eUB8y3z5wZRGQu9lH6ekQcw-Wk,6365
+verifiers/v1/tasksets/nemo_gym/taskset.py,sha256=ZBL73Tv-YsmY_XA49T6Vx7xRfJlfsk8QtM8WZ723ZPo,6332
 verifiers/v1/tasksets/nemo_gym/toolset.py,sha256=7iaRooFIfCN16wdSJbYFAOsc9q-haTivINO4S65UY6U,3715
 verifiers/v1/tasksets/openenv/__init__.py,sha256=G726meYYDplC3QE3AyjdSShq7FUcb4NkVqowuwPPuJk,303
-verifiers/v1/tasksets/openenv/taskset.py,sha256=1Wc9-5eN2HMKAwL34yTNex3zYOvXNXibF5wCROoMbFI,6075
+verifiers/v1/tasksets/openenv/taskset.py,sha256=PzoKgoaN5DpwY1LoWoS6p1yZdC08BP2q60Cqqbl6QSs,6046
 verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2LtcI7VDUHYPzRE,329
-verifiers/v1/tasksets/textarena/taskset.py,sha256=9fa_unlFiFZyuN4I02rW3x4qyRnFWXqwTUZTloQefu0,4433
+verifiers/v1/tasksets/textarena/taskset.py,sha256=zGslerxmH2ZT7FJdP-72R6a9p71kr2pNS3r5_xnq1lQ,4406
 verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
 verifiers/v1/utils/artifacts.py,sha256=i9DRhxViMAf-aBPjs3_9Q8RWVRC5pD-_v97VbNPPNcY,9638
@@ -192,8 +193,8 @@ verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,8
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/trace_store.py,sha256=RsCDonGR-bs-tvEstcppiJm3jFmAdIiG83fA5KtCbAw,2801
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev151.dist-info/METADATA,sha256=1VtFlk1XKVSzFTnSb4uqkTEl8Je4hFcEhWUo6h5r3RE,4161
-verifiers-0.3.2.dev151.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-verifiers-0.3.2.dev151.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.3.2.dev151.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev151.dist-info/RECORD,,
+verifiers-0.3.2.dev152.dist-info/METADATA,sha256=GPTXqNo6x_F7kc4NNULLjsI0ZcqMJjqU-l9xJqqxm48,4161
+verifiers-0.3.2.dev152.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+verifiers-0.3.2.dev152.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
+verifiers-0.3.2.dev152.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev152.dist-info/RECORD,,