verifiers 0.3.2.dev151__tar.gz → 0.3.2.dev153__tar.gz

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev151
+Version: 0.3.2.dev153
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/tests/v1/conftest.py

@@ -155,7 +155,10 @@ def _eval_config(
         runtime_cfg = dict(runtime)
         _configure_prime_runtimes(runtime_cfg)
         env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
-    retries = {"max_retries": 2, "include": ["ProviderError", "HarnessError"]}
+    retries = {
+        "max_retries": 2,
+        "include": ["ProviderError", "InterceptionError", "HarnessError"],
+    }
     env_cfg.setdefault("retries", retries)
     # Per-run caps live on the seats: resolve the env's declared roles and cap
     # each one (a test's own seat dict wins over the shared defaults).
@@ -177,7 +180,7 @@ def _eval_config(
             "taskset": taskset_cfg,
             **env_cfg,
         },
-        num_tasks=num_tasks,
+        select={"limit": num_tasks},
         num_rollouts=n,
         sampling={
             "max_tokens": max_tokens,
@@ -218,7 +221,7 @@ def run_v1_server():
 
     async def _run(taskset: str, **kwargs) -> list[Trace]:
         config = _eval_config(taskset, **kwargs)
-        tasks = list(load_taskset(config.env.taskset).head(config.num_tasks))
+        tasks = list(load_taskset(config.env.taskset).select(config.select))
         mpctx = mp.get_context("spawn")
         address_queue: mp.Queue = mpctx.Queue()
         proc = mpctx.Process(

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/tests/v1/test_color_codeword_tasks.py

@@ -10,7 +10,7 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
         return render(image)
 
     monkeypatch.setattr(taskset, "image_data_url", counted)
-    tasks = list(taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).head(100))
+    tasks = list(taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).take(100))
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_RGB.values())

verifiers-0.3.2.dev153/tests/v1/test_taskset.py

@@ -0,0 +1,201 @@
+"""`Taskset` iteration and selection: the framework-assigned `idx`, the lazy views
+(`include`, `exclude`, `shuffle`, `skip`, `take`), and `select(SelectConfig)`."""
+
+import itertools
+import logging
+import random
+
+import pytest
+from pydantic import ValidationError
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+def count_task(i: int) -> CountTask:
+    # A taskset-set idx (100 + i) that iteration must overwrite.
+    return CountTask(
+        vf.TaskData(idx=100 + i, id=f"id-{i}", name=f"t{i}", prompt=f"task {i}")
+    )
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        return [count_task(i) for i in range(10)]
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield count_task(i)
+
+
+def finite() -> FiniteTaskset:
+    return FiniteTaskset(vf.TasksetConfig())
+
+
+def infinite() -> InfiniteTaskset:
+    return InfiniteTaskset(vf.TasksetConfig())
+
+
+def idxs(tasks) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def shuffled(xs, seed: int = 0) -> list[int]:
+    xs = list(xs)
+    random.Random(seed).shuffle(xs)
+    return xs
+
+
+# Views
+
+
+def test_iteration_sets_idx_to_the_load_position() -> None:
+    assert idxs(finite()) == list(range(10))
+
+
+def test_include_keeps_tasks_that_any_list_names() -> None:
+    assert idxs(finite().include(idx=[3, "5:7", "8:"])) == [3, 5, 6, 8, 9]
+    assert idxs(finite().include(idx=":2")) == [0, 1]
+    assert idxs(finite().include(idx=["1:8:3", "::4"])) == [0, 1, 4, 7, 8]
+    assert idxs(finite().include(ids=["id-1"])) == [1]
+    assert idxs(finite().include(names=["t2"])) == [2]
+    key = list(finite())[4].key
+    assert idxs(finite().include(keys=[key])) == [4]
+    union = finite().include(idx=[0], ids=["id-1"], names=["t2"], keys=[key])
+    assert idxs(union) == [0, 1, 2, 4]
+
+
+def test_keys_match_the_keys_traces_record(tmp_path) -> None:
+    prompt = tmp_path / "prompt.txt"
+    prompt.write_text("be brief")
+    taskset = FiniteTaskset(vf.TasksetConfig(system_prompt=prompt))
+    key = list(taskset)[4].key
+    assert idxs(taskset.include(keys=[key])) == [4]
+    assert 4 not in idxs(taskset.exclude(keys=[key]))
+
+
+def test_exclude_drops_tasks_that_any_list_names() -> None:
+    assert idxs(finite().exclude(idx=":7", names=["t8"])) == [7, 9]
+
+
+def test_unmatched_entries_warn(caplog) -> None:
+    with caplog.at_level(logging.WARNING):
+        assert idxs(finite().include(names=["t1", "typo"], idx=[42])) == [1]
+    assert "typo" in caplog.text and "42" in caplog.text
+
+
+def test_shuffle_is_seeded_and_reproducible() -> None:
+    assert idxs(finite().shuffle()) == shuffled(range(10)) != list(range(10))
+    assert idxs(finite().shuffle(seed=7)) == shuffled(range(10), 7)
+
+
+def test_skip_and_take_slice_the_stream() -> None:
+    assert idxs(finite().skip(7)) == [7, 8, 9]
+    assert idxs(finite().take(3)) == [0, 1, 2]
+    assert idxs(finite().skip(2).take(2)) == [2, 3]
+
+
+def test_views_chain_in_call_order_and_leave_the_base_alone() -> None:
+    base = finite()
+    assert idxs(base.take(5).shuffle()) == shuffled(range(5))
+    assert idxs(base.shuffle().take(5)) == shuffled(range(10))[:5]
+    assert base.transform is None and idxs(base) == list(range(10))
+
+
+def test_views_read_load_lazily() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield count_task(i)
+
+    base = RecordingTaskset(vf.TasksetConfig())
+    assert idxs(base.take(3)) == [0, 1, 2] and built == [0, 1, 2]
+    built.clear()
+    # An idx-only include stops reading after its last position.
+    assert idxs(base.include(idx=["2:4", 6])) == [2, 3, 6] and built == list(range(7))
+
+
+def test_only_a_bounded_view_can_shuffle() -> None:
+    assert finite().bounded and not infinite().bounded
+    assert infinite().take(3).bounded
+    assert infinite().include(idx=["0:4", 9]).bounded
+    assert not infinite().include(idx="::2").bounded
+    assert not infinite().include(idx="5:").bounded
+    with pytest.raises(ValueError, match="may never end"):
+        infinite().include(idx=["0:4"], names=["t9"])
+    with pytest.raises(ValueError, match="open idx range"):
+        infinite().exclude(idx="5:")
+    assert idxs(infinite().exclude(idx="::2").take(2)) == [1, 3]
+    assert idxs(infinite().include(idx=":20").include(names=["t3"])) == [3]
+    with pytest.raises(ValueError, match="infinite"):
+        infinite().shuffle()
+    assert sorted(idxs(infinite().take(5).shuffle())) == [0, 1, 2, 3, 4]
+
+
+# Config and select
+
+
+def test_task_match_config_parses_idx() -> None:
+    assert vf.TaskMatchConfig(idx=["3", "0:2,4", "8:"]).idx == [3, "0:2", 4, "8:"]
+    assert vf.TaskMatchConfig(idx=5).idx == [5]
+    assert vf.TaskMatchConfig(idx="0:9:3").idx_stop == 9
+    for bad in [-1, "a", "5:5", "7:3", ":0", "1:2:0", "::00", "1:2:3:4"]:
+        with pytest.raises(ValidationError):
+            vf.TaskMatchConfig(idx=[bad])
+
+
+def test_select_config_defaults_and_short_aliases() -> None:
+    config = vf.SelectConfig()
+    assert config.include.empty and config.exclude.empty
+    assert (config.shuffle, config.seed, config.skip, config.limit) == (
+        False,
+        0,
+        0,
+        None,
+    )
+    aliased = vf.SelectCLIConfig.model_validate({"n": 3, "s": True})
+    assert (aliased.limit, aliased.shuffle) == (3, True)
+    with pytest.raises(ValidationError):
+        vf.SelectConfig.model_validate({"n": 3})
+    with pytest.raises(ValidationError):
+        vf.SelectConfig(limit=0)
+
+
+def test_default_select_yields_every_task() -> None:
+    assert idxs(finite().select(vf.SelectConfig())) == list(range(10))
+
+
+def test_select_applies_its_steps_in_a_fixed_order() -> None:
+    config = vf.SelectConfig(
+        include={"idx": ["2:9"]},
+        exclude={"names": ["t5"]},
+        shuffle=True,
+        seed=3,
+        skip=1,
+        limit=2,
+    )
+    kept = [i for i in range(2, 9) if i != 5]
+    assert idxs(finite().select(config)) == shuffled(kept, 3)[1:3]
+    chained = (
+        finite().include(idx=["2:9"]).exclude(names=["t5"]).shuffle(3).skip(1).take(2)
+    )
+    assert idxs(finite().select(config)) == idxs(chained)
+
+
+def test_select_bounds_an_infinite_taskset() -> None:
+    assert infinite().select(vf.SelectConfig(limit=4)).bounded
+    closed = vf.SelectConfig(include={"idx": "0:5"}, shuffle=True)
+    assert sorted(idxs(infinite().select(closed))) == [0, 1, 2, 3, 4]
+    # The shuffle comes before the limit, so the limit cannot bound it.
+    with pytest.raises(ValueError, match="infinite"):
+        infinite().select(vf.SelectConfig(shuffle=True, limit=5))

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/__init__.py

@@ -23,6 +23,7 @@ from verifiers.v1.configs.env import EnvConfig, SharedEnvConfig, default_agent_h
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig, Judges
 from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.select import SelectCLIConfig, SelectConfig, TaskMatchConfig
 from verifiers.v1.configs.serve import (
     ElasticPoolConfig,
     ServeConfig,
@@ -289,6 +290,9 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     # taskset / harness / runtime / environment
     "Taskset",
     "TaskConfig",
+    "SelectConfig",
+    "SelectCLIConfig",
+    "TaskMatchConfig",
     "TasksetConfig",
     "SharedTasksetConfig",
     "DecoratedFunctionConfig",

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/acp/__init__.py

@@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.errors import HarnessError
+from verifiers.v1.errors import HarnessError, InterceptionError
 from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess
 from verifiers.v1.semantic import (
@@ -308,6 +308,12 @@ class ACPHarnessSession(HarnessSession):
             detail = response.get("error") or "ACP session request failed"
             if stderr := self._stderr():
                 detail = f"{detail}\n\nACP process stderr:\n{stderr}"
+            error_data = response.get("error_data")
+            if (
+                isinstance(error_data, dict)
+                and error_data.get("kind") == "model_transport"
+            ):
+                raise InterceptionError(detail)
             raise RuntimeError(detail)
         harness = cast(ACPHarness, self.harness)
         harness._consume_protocol_metadata(self.trace, turn.response_metadata)

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/acp/runner.py

@@ -224,9 +224,6 @@ async def prompt(
         if client.stop_reason != "cancelled":
             raise
         await connection.cancel(session_id=session_id)
-    except RequestError as error:
-        detail = error.data.get("details") if isinstance(error.data, dict) else None
-        raise RuntimeError(detail or str(error)) from error
     finally:
         client.prompt_task = None
     return client.turn_result()
@@ -377,6 +374,9 @@ async def serve_stream() -> None:
                     "ok": False,
                     "error": f"{type(error).__name__}: {error}",
                 }
+                if isinstance(error, RequestError) and isinstance(error.data, dict):
+                    response["error"] = error.data.get("details") or str(error)
+                    response["error_data"] = error.data
                 if operation == "prompt":
                     response["result"] = asdict(session.client.turn_result())
             write_packet(sys.stdout.buffer, response)

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/cli/debug.py

@@ -269,15 +269,12 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
-    taskset = vf.load_taskset(config.taskset)
-    if config.num_tasks is None and taskset.INFINITE:
+    taskset = vf.load_taskset(config.taskset).select(config.select)
+    if not taskset.bounded:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
         )
-    selected = taskset.shuffle() if config.shuffle else taskset
-    if config.num_tasks is not None:
-        selected = selected.head(config.num_tasks)
-    tasks = list(selected)
+    tasks = list(taskset)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/cli/eval/runner.py

@@ -82,15 +82,12 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
     from verifiers.v1.utils.loaders import load_environment
 
     env = load_environment(config.env)
-    taskset = env.taskset
-    if config.num_tasks is None and taskset.INFINITE:
+    taskset = env.taskset.select(config.select)
+    if not taskset.bounded:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
         )
-    selected = taskset.shuffle() if config.shuffle else taskset
-    if config.num_tasks is not None:
-        selected = selected.head(config.num_tasks)
-    tasks = list(selected)
+    tasks = list(taskset)
     out = output_path(config)
     # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts.
     plan = [(task, config.num_rollouts) for task in tasks]

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/cli/init.py

@@ -109,7 +109,7 @@ class {prefix}Taskset(vf.Taskset[{prefix}Task, {prefix}Config]):
     def load(self) -> list[{prefix}Task]:
         raise NotImplementedError(
             "Return this taskset's tasks, e.g. "
-            "[{prefix}Task({prefix}Data(idx=i, prompt=...), self.config.task) "
+            "[{prefix}Task({prefix}Data(prompt=...), self.config.task) "
             "for i in range(self.config.num_tasks)]."
         )
 '''

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/cli/validate.py

@@ -325,15 +325,12 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
-    taskset = vf.load_taskset(config.taskset)
-    if config.num_tasks is None and taskset.INFINITE:
+    taskset = vf.load_taskset(config.taskset).select(config.select)
+    if not taskset.bounded:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
         )
-    selected = taskset.shuffle() if config.shuffle else taskset
-    if config.num_tasks is not None:
-        selected = selected.head(config.num_tasks)
-    tasks = list(selected)
+    tasks = list(taskset)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/configs/cli/debug.py

@@ -7,6 +7,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.cli.validate import CheckTimeoutConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
@@ -15,6 +16,9 @@ class DebugConfig(BaseConfig):
     uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
     """Auto-generated run id, used as the default output directory leaf."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks to debug, under `--select.*` (`-n` sets
+    `select.limit`, `-s` sets `select.shuffle`)."""
     runtime: RuntimeConfig = PrimeConfig()
     """Where each task's setup hook and debug action run."""
     command: str | None = None
@@ -26,14 +30,6 @@ class DebugConfig(BaseConfig):
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     """Per-task stage timeouts: `--timeout.setup` for the `setup` hook, `--timeout.total`
     for the debug command/script."""
-    num_tasks: int | None = Field(
-        None,
-        ge=1,
-        validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
-    )
-    """How many tasks to debug (None = all)."""
-    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
         128, validation_alias=AliasChoices("max_concurrent", "c")
     )

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/configs/cli/eval.py

@@ -9,6 +9,7 @@ from pydantic_config import BaseConfig
 from verifiers.v1.clients import ClientConfig, EvalClientConfig
 from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
 from verifiers.v1.configs.env import EnvConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
 
@@ -84,12 +85,9 @@ class EvalConfig(BaseConfig):
     """Model id."""
     client: ClientConfig = EvalClientConfig()
     sampling: SamplingConfig = SamplingConfig()
-    num_tasks: int | None = Field(
-        None,
-        ge=1,
-        validation_alias=AliasChoices("batch_size", "num_examples", "num_tasks", "n"),
-    )
-    """How many tasks to evaluate (None = all)."""
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks to evaluate, under `--select.*` (`-n` sets
+    `select.limit`, `-s` sets `select.shuffle`)."""
     num_rollouts: int = Field(
         1,
         ge=1,
@@ -98,8 +96,6 @@ class EvalConfig(BaseConfig):
         ),
     )
     """Independent episodes per task — the trainer's group size."""
-    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
         128, ge=1, validation_alias=AliasChoices("max_concurrent", "c")
     )

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/configs/cli/validate.py

@@ -7,6 +7,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.cli.eval import RunConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
@@ -24,6 +25,9 @@ class ValidateConfig(BaseConfig):
     """Run identity: `run.name` auto-generates as `<taskset>--validate--<short-id>` and
     names the run directory under `output_dir`."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks to validate, under `--select.*` (`-n` sets
+    `select.limit`, `-s` sets `select.shuffle`)."""
     runtime: RuntimeConfig = PrimeConfig()
     """Where each task's validation hooks run."""
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
@@ -31,14 +35,6 @@ class ValidateConfig(BaseConfig):
     """Run only `Task.setup`."""
     only_gold: bool = False
     """Run only `Task.setup` and `Task.validate`."""
-    num_tasks: int | None = Field(
-        None,
-        ge=1,
-        validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
-    )
-    """How many tasks to validate (None = all)."""
-    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
         128, validation_alias=AliasChoices("max_concurrent", "c")
     )

verifiers-0.3.2.dev153/verifiers/v1/configs/select.py

@@ -0,0 +1,118 @@
+"""Which of a taskset's tasks a run uses: `Taskset.select(SelectConfig)`, under
+`--select.*` on the eval, debug, validate and GEPA CLIs (`SelectCLIConfig`)."""
+
+import re
+import sys
+
+from pydantic import AliasChoices, Field, field_validator
+from pydantic_config import BaseConfig
+
+IDX_RANGE = re.compile(r"(\d*):(\d*)(?::(\d*))?")
+
+
+class TaskMatchConfig(BaseConfig):
+    """Tasks named by position or identity; a task matches when any list names it."""
+
+    idx: list[int | str] = Field(default_factory=list)
+    """Positions in the taskset's `load()` stream (`TaskData.idx`): ints and Python
+    slices `start:stop:step`, each part optional (`"100:"`, `":50"`, `"::2"`).
+    Negative positions are not supported. A comma-separated string also works
+    (`"0:10,17"`)."""
+    ids: list[str] = Field(default_factory=list)
+    """`TaskData.id` values."""
+    keys: list[str] = Field(default_factory=list)
+    """`Task.key` values, as recorded on each trace (`task.key`)."""
+    names: list[str] = Field(default_factory=list)
+    """`TaskData.name` values."""
+
+    @field_validator("idx", mode="before")
+    @classmethod
+    def _parse_idx(cls, value):
+        if isinstance(value, (int, str)):
+            value = [value]
+        items: list[int | str] = []
+        for item in value:
+            parts = item.split(",") if isinstance(item, str) else [item]
+            items.extend(_parse_idx_item(part) for part in parts)
+        return items
+
+    @property
+    def empty(self) -> bool:
+        return not (self.idx or self.ids or self.keys or self.names)
+
+    def idx_ranges(self) -> list[range]:
+        """`idx` as ranges; an open-ended slice stops at `sys.maxsize`."""
+        ranges: list[range] = []
+        for item in self.idx:
+            if isinstance(item, int):
+                ranges.append(range(item, item + 1))
+                continue
+            start, stop, step = (item.split(":") + [""])[:3]
+            ranges.append(
+                range(int(start or 0), int(stop or sys.maxsize), int(step or 1))
+            )
+        return ranges
+
+    @property
+    def idx_tail(self) -> bool:
+        """Whether an `idx` slice names every position from some start on."""
+        return any(r.stop == sys.maxsize and r.step == 1 for r in self.idx_ranges())
+
+    @property
+    def idx_stop(self) -> int | None:
+        """One past the last position this can match, when it names only closed `idx`
+        ranges; None when a match could lie anywhere in the stream."""
+        ranges = self.idx_ranges()
+        if not ranges or self.ids or self.keys or self.names:
+            return None
+        stop = max(r.stop for r in ranges)
+        return None if stop == sys.maxsize else stop
+
+
+def _parse_idx_item(item: int | str) -> int | str:
+    if isinstance(item, int):
+        if item < 0:
+            raise ValueError(f"idx {item} is negative")
+        return item
+    text = item.strip()
+    if text.isdigit():
+        return int(text)
+    match = IDX_RANGE.fullmatch(text)
+    if match is None:
+        raise ValueError(f"idx {item!r} is not an int or a 'start:stop:step' slice")
+    start, stop, step = match.groups()
+    if step and int(step) == 0:
+        raise ValueError(f"idx slice {item!r} has step 0")
+    if stop and int(start or 0) >= int(stop):
+        raise ValueError(f"idx range {item!r} is empty")
+    return text
+
+
+class SelectConfig(BaseConfig):
+    """The steps apply in a fixed order: `include` keeps tasks, `exclude` drops them,
+    then `shuffle`, `skip` and `limit` pick from what is left."""
+
+    include: TaskMatchConfig = TaskMatchConfig()
+    """Keep only the tasks this names. Empty keeps all."""
+    exclude: TaskMatchConfig = TaskMatchConfig()
+    """Drop the tasks this names."""
+    shuffle: bool = False
+    """Shuffle the kept tasks under `seed`. Needs a finite stream: an infinite taskset
+    must be bounded by closed `include.idx` ranges first."""
+    seed: int = 0
+    """Seed for `shuffle`, fixed so runs select the same tasks."""
+    skip: int = Field(0, ge=0)
+    """Drop this many tasks after the shuffle."""
+    limit: int | None = Field(None, ge=1)
+    """Take at most this many tasks after `skip` (None = all)."""
+
+
+class SelectCLIConfig(SelectConfig):
+    """`SelectConfig` with CLI short flags, for the one `select` block of an entrypoint:
+    `-n` sets `limit` and `-s` sets `shuffle`."""
+
+    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
+    """Shuffle the kept tasks under `seed` (`-s`). Needs a finite stream: an infinite
+    taskset must be bounded by closed `include.idx` ranges first."""
+    limit: int | None = Field(None, ge=1, validation_alias=AliasChoices("limit", "n"))
+    """Take at most this many tasks after `skip` (`-n`; None = all)."""

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/errors.py

@@ -42,6 +42,11 @@ class ProviderError(RolloutError):
         self.status_code = status_code
 
 
+MODEL_TRANSPORT_ERROR_EXIT_CODE = 97
+"""Reserved harness exit code for an exhausted model transport failure.
+`Harness._check_result` maps it to `InterceptionError`."""
+
+
 class HarnessError(RolloutError):
     """The harness failed to install or launch, or its agent process exited unsuccessfully."""
 
@@ -65,7 +70,7 @@ class TaskError(RolloutError):
 
 
 class InterceptionError(RolloutError):
-    """The host interception server (model calls + `/state` + `/task` channels) couldn't be reached."""
+    """Communication with the host interception server failed."""
 
 
 class TunnelError(InterceptionError):

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/gepa/config.py

@@ -17,6 +17,7 @@ from verifiers.v1.clients import EvalClientConfig
 from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
 from verifiers.v1.configs.cli.eval import RunConfig, default_run_name
 from verifiers.v1.configs.env import EnvConfig
+from verifiers.v1.configs.select import SelectCLIConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
 
@@ -49,17 +50,17 @@ class GEPAConfig(BaseConfig):
     reflection_client: EvalClientConfig | None = None
     """Endpoint for `reflection_model`. None = reuse `client`."""
 
+    select: SelectCLIConfig = SelectCLIConfig()
+    """Which of the taskset's tasks GEPA splits into train/val, under `--select.*`
+    (`-n` sets `select.limit`, `-s` sets `select.shuffle`)."""
     num_train: int = Field(100, ge=1)
     """Tasks reserved for reflection minibatches (GEPA never scores the full trainset at once)."""
     num_val: int = Field(50, ge=1)
     """Tasks held out to score each candidate system prompt for the pareto frontier."""
-    shuffle: bool = Field(True, validation_alias=AliasChoices("shuffle", "s"))
-    """Shuffle tasks before splitting into train/val — v1 tasksets have no generic train/val
-    split, so GEPA carves one out of `Taskset.select` the way `run_eval` samples (fixed
-    seed, so the split is reproducible across runs)."""
     seed: int = 0
-    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling
-    uses a fixed seed, matching eval — so this doesn't change the train/val split."""
+    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). The train/val
+    split comes from `select` (`-s` shuffles it under `select.seed`), so this doesn't
+    change it."""
 
     max_total_rollouts: int = Field(500)
     """Total rollouts GEPA may spend across the whole optimization run."""

{verifiers-0.3.2.dev151 → verifiers-0.3.2.dev153}/verifiers/v1/gepa/dataset.py

@@ -2,8 +2,8 @@
 
 v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
 individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of the tasks
-`Taskset.select` hands it (the shared fixed-seed shuffle, reproducible across runs like every
-other entrypoint): two disjoint slices.
+`select` yields (reproducible across runs like every other entrypoint): two
+disjoint slices.
 """
 
 from verifiers.v1.task import Task