verifiers 0.2.2.dev50__tar.gz → 0.2.2.dev52__tar.gz

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev50
+Version: 0.2.2.dev52
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/tests/v1/conftest.py

@@ -135,10 +135,6 @@ def _eval_config(
     `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
     harness one); `model` overrides the default text model (e.g. a VLM for an image task).
 
-    `temperature=0` (greedy) makes the run reproducible; `max_tokens` is generous headroom,
-    not a target — these trivial tasks finish in a few hundred tokens, so capping tighter only
-    risks truncating the reasoning before the answer (which tanks the reward).
-
     `harness=None` leaves every seat on its own story — the multi-agent case: there
     is no run-level harness, so a single-agent test's `harness` lands on the `agent`
     seat and a multi-agent test pins its seats through `env` role fields instead."""
@@ -175,7 +171,6 @@ def _eval_config(
         num_rollouts=n,
         sampling={
             "max_tokens": max_tokens,
-            "temperature": 0,
             "reasoning_effort": reasoning_effort,
         },
         rich=False,
@@ -219,8 +214,8 @@ def run_v1_server():
 
 @pytest.fixture
 async def live_ctx():
-    """A live `ModelContext` (the e2e default model + endpoint, greedy) for driving
-    `Agent` directly — the agent-surface counterpart of `run_v1`."""
+    """A live `ModelContext` (the e2e default model + endpoint, provider-default
+    sampling) for driving `Agent` directly — the agent-surface counterpart of `run_v1`."""
     from verifiers.v1.clients import EvalClientConfig, ModelContext, resolve_client
     from verifiers.v1.types import SamplingConfig
 
@@ -229,7 +224,7 @@ async def live_ctx():
         yield ModelContext(
             model="deepseek/deepseek-v4-flash",
             client=client,
-            sampling=SamplingConfig(max_tokens=2048, temperature=0),
+            sampling=SamplingConfig(max_tokens=2048),
         )
     finally:
         await client.close()
@@ -252,7 +247,7 @@ def run_v0():
             legacy={"id": env_id, "args": args or {}},
             num_tasks=1,
             num_rollouts=n,
-            sampling={"max_tokens": max_tokens, "temperature": 0},
+            sampling={"max_tokens": max_tokens},
             rich=False,
             output_dir=output_dir,
         )

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/tests/v1/test_color_codeword_tasks.py

@@ -10,9 +10,7 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
         return render(image)
 
     monkeypatch.setattr(taskset, "image_data_url", counted)
-    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
-        num_tasks=100
-    )
+    tasks = list(taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).head(100))
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_RGB.values())

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/tests/v1/test_e2e.py

@@ -109,7 +109,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
     # The seat's resolved identity rides the trace (policy metadata for trainers).
-    assert trace.agent is not None and trace.agent.config.sampling.temperature == 0
+    assert trace.agent is not None and trace.agent.config.sampling.max_tokens == 2048
     # Every sampled turn has one per-call record, linked to its assistant node.
     sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
     assert [c.node for c in trace.calls if c.error is None] == sampled

verifiers-0.2.2.dev52/tests/v1/test_taskset.py

@@ -0,0 +1,93 @@
+"""`Taskset` iteration and the `head`/`shuffle` views over list, generator,
+and `INFINITE` `load` implementations."""
+
+import itertools
+
+import pytest
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        for i in range(10):
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+def idxs(tasks) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def test_head_bounds_an_infinite_taskset() -> None:
+    tasks = list(InfiniteTaskset(vf.TasksetConfig()).head(5))
+    assert idxs(tasks) == [0, 1, 2, 3, 4]
+
+
+def test_iteration_materializes_a_finite_taskset() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    assert len(list(taskset)) == 10
+    assert idxs(taskset.head(4)) == [0, 1, 2, 3]
+
+
+def test_head_returns_the_same_taskset_type() -> None:
+    view = FiniteTaskset(vf.TasksetConfig()).head(4)
+    assert isinstance(view, FiniteTaskset)
+    assert view.task_type() is CountTask
+
+
+def test_shuffle_samples_the_whole_taskset_reproducibly() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    first = idxs(taskset.shuffle().head(5))
+    assert first == idxs(taskset.shuffle().head(5))
+    assert len(first) == 5 and set(first) <= set(range(10))
+    assert first != [0, 1, 2, 3, 4]  # sampled from the whole set, not the head
+
+
+def test_shuffle_seed_changes_the_sample() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    seeded = idxs(taskset.shuffle(seed=7).head(5))
+    assert seeded == idxs(taskset.shuffle(seed=7).head(5))  # reproducible per seed
+    assert seeded != idxs(taskset.shuffle().head(5))  # differs from the default seed
+
+
+def test_shuffle_raises_on_infinite() -> None:
+    with pytest.raises(ValueError, match="infinite"):
+        InfiniteTaskset(vf.TasksetConfig()).shuffle()
+
+
+def test_head_then_shuffle_bounds_an_infinite_taskset() -> None:
+    head = idxs(InfiniteTaskset(vf.TasksetConfig()).head(5).shuffle())
+    assert sorted(head) == [0, 1, 2, 3, 4]
+    assert head != [0, 1, 2, 3, 4]  # shuffled within the bounded head
+
+
+def test_head_only_builds_what_the_run_takes() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+    list(RecordingTaskset(vf.TasksetConfig()).head(3))
+    assert built == [0, 1, 2]
+
+
+def test_views_do_not_mutate_the_base_taskset() -> None:
+    taskset = InfiniteTaskset(vf.TasksetConfig())
+    view = taskset.head(3)
+    assert view.INFINITE is False and taskset.INFINITE is True
+    assert idxs(taskset.head(2)) == [0, 1]  # base iterates untransformed

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/cli/debug.py

@@ -263,7 +263,14 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.select(config.num_tasks, config.shuffle)
+    if config.num_tasks is None and taskset.INFINITE:
+        raise ValueError(
+            f"{type(taskset).__name__} is infinite - bound the run with -n"
+        )
+    selected = taskset.shuffle() if config.shuffle else taskset
+    if config.num_tasks is not None:
+        selected = selected.head(config.num_tasks)
+    tasks = list(selected)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/cli/eval/runner.py

@@ -26,7 +26,15 @@ logger = logging.getLogger(__name__)
 async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     client = resolve_client(config.client)
-    tasks = env.taskset.select(config.num_tasks, config.shuffle)
+    taskset = env.taskset
+    if config.num_tasks is None and taskset.INFINITE:
+        raise ValueError(
+            f"{type(taskset).__name__} is infinite - bound the run with -n"
+        )
+    selected = taskset.shuffle() if config.shuffle else taskset
+    if config.num_tasks is not None:
+        selected = selected.head(config.num_tasks)
+    tasks = list(selected)
     ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     semaphore = (
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
@@ -132,9 +140,15 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
 
         # The client owns the taskset: load it here, once — the server (and its pool
         # workers) never load data, they rebuild each dispatched task from its request.
-        tasks = load_taskset(config.env.taskset).select(
-            config.num_tasks, config.shuffle
-        )
+        taskset = load_taskset(config.env.taskset)
+        if config.num_tasks is None and taskset.INFINITE:
+            raise ValueError(
+                f"{type(taskset).__name__} is infinite - bound the run with -n"
+            )
+        selected = taskset.shuffle() if config.shuffle else taskset
+        if config.num_tasks is not None:
+            selected = selected.head(config.num_tasks)
+        tasks = list(selected)
     # Spawned processes inherit no logging — hand them the main process's setup so
     # their rollout logs land in the output dir.
     level = "DEBUG" if config.verbose else "INFO"

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/cli/validate.py

@@ -203,7 +203,14 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.select(config.num_tasks, config.shuffle)
+    if config.num_tasks is None and taskset.INFINITE:
+        raise ValueError(
+            f"{type(taskset).__name__} is infinite - bound the run with -n"
+        )
+    selected = taskset.shuffle() if config.shuffle else taskset
+    if config.num_tasks is not None:
+        selected = selected.head(config.num_tasks)
+    tasks = list(selected)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/configs/taskset.py

@@ -17,8 +17,8 @@ class TasksetConfig(BaseConfig):
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
     system_prompt: Path | None = None
-    """File whose text overrides each task's `TaskData.system_prompt` in
-    `Taskset.select` (e.g. a GEPA `best_system_prompt.txt`)."""
+    """File whose text overrides each task's `TaskData.system_prompt` on
+    iteration (e.g. a GEPA `best_system_prompt.txt`)."""
 
     @property
     def name(self) -> str:

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/env.py

@@ -33,7 +33,7 @@ from verifiers.v1.retries import run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.task import Task, resolve_server_config
 from verifiers.v1.trace import Error, Trace
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
 
 logger = logging.getLogger(__name__)
@@ -83,7 +83,7 @@ class Env(ABC, Generic[ConfigT]):
     def __init__(self, config: ConfigT) -> None:
         from verifiers.v1.loaders import load_harness, load_taskset
 
-        config_cls = generic_type(type(self), EnvConfig, origin=Env) or EnvConfig
+        config_cls = concrete_type(type(self), EnvConfig, origin=Env) or EnvConfig
         if not isinstance(config, config_cls):
             raise TypeError(
                 f"{type(self).__name__} declares Env[{config_cls.__name__}], "

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/gepa/runner.py

@@ -37,7 +37,10 @@ class _GEPALog:
 
 def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
     logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
-    all_tasks = env.taskset.select(config.num_train + config.num_val, config.shuffle)
+    # Global shuffle: an infinite taskset raises here — run it with
+    # shuffle=false (there is no whole set to sample from).
+    taskset = env.taskset.shuffle() if config.shuffle else env.taskset
+    all_tasks = list(taskset.head(config.num_train + config.num_val))
     train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val)
     selected_tasks = [*train_tasks, *val_tasks]
     # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool —

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/judge.py

@@ -63,7 +63,7 @@ from verifiers.v1.configs.judge import (
 from verifiers.v1.dialects.chat import message_to_wire
 from verifiers.v1.scoring import parse_judge_choice
 from verifiers.v1.types import Messages, StrictBaseModel, Usage
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
     from verifiers.v1.task import TaskData
@@ -111,7 +111,7 @@ ConfigT = TypeVar("ConfigT", bound=JudgeConfig, default=JudgeConfig)
 
 def judge_config_cls(cls: type) -> type[JudgeConfig]:
     """Resolve a judge's config specialization through its MRO, else `JudgeConfig`."""
-    return generic_type(cls, JudgeConfig) or JudgeConfig
+    return concrete_type(cls, JudgeConfig) or JudgeConfig
 
 
 class Judge(Generic[ParsedT, ConfigT]):

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/loaders.py

@@ -20,7 +20,7 @@ from verifiers.v1.harness import Harness
 from verifiers.v1.judge import Judge, judge_config_cls
 from verifiers.v1.task import Task
 from verifiers.v1.taskset import Taskset
-from verifiers.v1.utils.generic import generic_type, prefix_validation_error
+from verifiers.v1.utils.generic import concrete_type, prefix_validation_error
 from verifiers.v1.utils.install import ensure_installed
 
 
@@ -208,7 +208,7 @@ def load_judge(config: JudgeConfig) -> Judge:
 def taskset_config_type(taskset_id: str) -> type[TasksetConfig]:
     """Resolve the taskset's config specialization through its MRO."""
     return (
-        generic_type(taskset_class(taskset_id), TasksetConfig, origin=Taskset)
+        concrete_type(taskset_class(taskset_id), TasksetConfig, origin=Taskset)
         or TasksetConfig
     )
 
@@ -216,7 +216,7 @@ def taskset_config_type(taskset_id: str) -> type[TasksetConfig]:
 def harness_config_type(harness_id: str) -> type[HarnessConfig]:
     """Resolve the harness's config specialization through its MRO."""
     return (
-        generic_type(harness_class(harness_id), HarnessConfig, origin=Harness)
+        concrete_type(harness_class(harness_id), HarnessConfig, origin=Harness)
         or HarnessConfig
     )
 
@@ -231,7 +231,7 @@ def env_config_type(taskset_id: str, env_id: str = "") -> type[EnvConfig]:
     its MRO — `SingleAgentEnvConfig` for a plain taskset. The run's `env` field
     narrows to this, which is what gives `--env.<role>.model` addressing."""
     return (
-        generic_type(environment_class(taskset_id, env_id), EnvConfig, origin=Env)
+        concrete_type(environment_class(taskset_id, env_id), EnvConfig, origin=Env)
         or EnvConfig
     )
 

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/mcp/server.py

@@ -13,7 +13,7 @@ from pydantic import TypeAdapter, ValidationError
 from pydantic_config import BaseConfig
 
 from verifiers.v1.state import State, StateT, state_cls
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
     from httpx import AsyncClient, Response
@@ -286,7 +286,7 @@ class ServerBase(Generic[ConfigT, StateT]):
     @classmethod
     def _config_cls(cls) -> type[BaseConfig]:
         """Resolve the server's config specialization through its MRO."""
-        if config_cls := generic_type(cls, BaseConfig):
+        if config_cls := concrete_type(cls, BaseConfig):
             return config_cls
         raise TypeError(
             f"{cls.__name__} must parameterize its config, e.g. Toolset[MyConfig]"

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/state.py

@@ -8,7 +8,7 @@ from pydantic import ConfigDict
 from typing_extensions import TypeVar
 
 from verifiers.v1.types import StrictBaseModel
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 
 class State(StrictBaseModel):
@@ -20,4 +20,4 @@ StateT = TypeVar("StateT", bound=State, default=State)
 
 def state_cls(cls: type) -> type[State]:
     """Resolve a class's `State` specialization through its MRO, else `State`."""
-    return generic_type(cls, State) or State
+    return concrete_type(cls, State) or State

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/task.py

@@ -39,7 +39,7 @@ from verifiers.v1.decorators import discover_decorated, invoke_all
 from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, StrictBaseModel, content_text
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
     from verifiers.v1.judge import Judge
@@ -50,27 +50,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _requires_runtime(fn) -> bool:
-    param = inspect.signature(fn).parameters.get("runtime")
-    # A defaulted runtime parameter can still be called offline with None.
-    return param is not None and param.default is inspect.Parameter.empty
-
-
-def _record_result(
-    trace: Trace,
-    name: str,
-    result,
-    weight: float | None = None,
-) -> None:
-    """Record a scalar or keyed scoring result."""
-    values = list(result.items()) if isinstance(result, Mapping) else [(name, result)]
-    for key, value in values:
-        if weight is None:
-            trace.record_metric(key, value)
-        else:
-            trace.record_reward(key, value, weight)
-
-
 class TaskResources(StrictBaseModel):
     model_config = ConfigDict(frozen=True)
 
@@ -149,12 +128,12 @@ ConfigT = TypeVar("ConfigT", bound=TaskConfig, default=TaskConfig)
 
 def task_data_cls(cls: type) -> type[TaskData]:
     """Resolve a task's `TaskData` specialization through its MRO, else `TaskData`."""
-    return generic_type(cls, TaskData) or TaskData
+    return concrete_type(cls, TaskData) or TaskData
 
 
 def task_config_cls(cls: type) -> type[TaskConfig]:
     """Resolve a task's `TaskConfig` specialization through its MRO, else `TaskConfig`."""
-    return generic_type(cls, TaskConfig) or TaskConfig
+    return concrete_type(cls, TaskConfig) or TaskConfig
 
 
 def resolve_server_config(
@@ -239,6 +218,11 @@ class Task(Generic[DataT, StateT, ConfigT]):
         trace: Trace,
         runtime: Runtime | None = None,
     ) -> None:
+        def requires_runtime(fn) -> bool:
+            param = inspect.signature(fn).parameters.get("runtime")
+            # A defaulted runtime parameter can still be called offline with None.
+            return param is not None and param.default is inspect.Parameter.empty
+
         judges = self.plugged_judges()
         available = {"task": self.data, "trace": trace}
         if runtime is not None:
@@ -249,36 +233,50 @@ class Task(Generic[DataT, StateT, ConfigT]):
             rewards = discover_decorated(self, "reward")
             if runtime is None:
                 skipped = [
-                    fn.__name__ for fn in (*metrics, *rewards) if _requires_runtime(fn)
+                    fn.__name__ for fn in (*metrics, *rewards) if requires_runtime(fn)
                 ] + [
                     judge.reward_name
                     for judge in judges
-                    if _requires_runtime(judge.score)
+                    if requires_runtime(judge.score)
                 ]
                 if skipped:
                     logger.info(
                         "score: no runtime — skipped runtime-dependent signals: %s",
                         skipped,
                     )
-                metrics = [fn for fn in metrics if not _requires_runtime(fn)]
-                rewards = [fn for fn in rewards if not _requires_runtime(fn)]
+                metrics = [fn for fn in metrics if not requires_runtime(fn)]
+                rewards = [fn for fn in rewards if not requires_runtime(fn)]
                 judges = [
-                    judge for judge in judges if not _requires_runtime(judge.score)
+                    judge for judge in judges if not requires_runtime(judge.score)
                 ]
 
             metric_results = await invoke_all(metrics, available)
             for fn, result in zip(metrics, metric_results):
-                _record_result(trace, fn.__name__, result)
+                if isinstance(result, Mapping):
+                    trace.record_metrics(result)
+                else:
+                    trace.record_metric(fn.__name__, result)
             reward_results = await invoke_all(rewards, available)
             for fn, result in zip(rewards, reward_results):
-                _record_result(
-                    trace, fn.__name__, result, getattr(fn, "_vf_weight", 1.0)
+                weight = getattr(fn, "_vf_weight", 1.0)
+                items = (
+                    result.items()
+                    if isinstance(result, Mapping)
+                    else [(fn.__name__, result)]
                 )
+                for key, value in items:
+                    trace.record_reward(key, value, weight)
             judge_results = await invoke_all(
                 [judge.score for judge in judges], available
             )
             for judge, result in zip(judges, judge_results):
-                _record_result(trace, judge.reward_name, result, judge.config.weight)
+                items = (
+                    result.items()
+                    if isinstance(result, Mapping)
+                    else [(judge.reward_name, result)]
+                )
+                for key, value in items:
+                    trace.record_reward(key, value, judge.config.weight)
 
 
 TaskT = TypeVar("TaskT", bound=Task)

verifiers-0.2.2.dev52/verifiers/v1/taskset.py

@@ -0,0 +1,116 @@
+"""The taskset: a thin loader that yields typed tasks.
+
+A `Taskset` is the data half of an environment: config in, tasks out. `load()` is
+the main hook that builds each task:
+
+    def load(self) -> Iterable[MyTask]:
+        for i in ...:
+            yield MyTask(MyData(idx=i, ...), self.config.task)
+
+`load` may also be a generator for infinite tasksets. There is a one-to-one
+mapping between taskset and task type, i.e. a taskset may only yield one task
+type.
+"""
+
+from __future__ import annotations
+
+import copy
+import itertools
+import random
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterable, Iterator
+from typing import TYPE_CHECKING, ClassVar, Generic, Self
+
+from pydantic_config import BaseConfig
+from typing_extensions import TypeVar
+
+from verifiers.v1.configs.taskset import TasksetConfig
+from verifiers.v1.task import Task, TaskT, resolve_server_config
+from verifiers.v1.utils.generic import concrete_type
+from verifiers.v1.utils.sampling import SEED
+
+if TYPE_CHECKING:
+    from verifiers.v1.mcp import Toolset
+
+TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetConfig)
+
+
+class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
+    INFINITE: bool = False
+    """Whether the taskset is infinite (yields tasks forever). Class-declared;
+    a `head(n)` view shadows it per instance (bounded by construction)."""
+
+    tools: ClassVar[tuple[type[Toolset], ...]] = ()
+    """Tool servers shared by all tasks in the taskset. The environment will
+    spawn a single, global instance, reused across tasks."""
+
+    def __init__(self, config: TasksetConfigT) -> None:
+        self.config = config
+        override = config.system_prompt
+        self.system_prompt = override.read_text() if override is not None else None
+        self.transform: Callable[[Iterator[TaskT]], Iterator[TaskT]] | None = None
+        """Iteration transform carried by `head`/`shuffle` views (see `view`)."""
+
+    @abstractmethod
+    def load(self) -> Iterable[TaskT]:
+        """Build and yield the taskset's tasks; may be a generator (see module doc)."""
+
+    def __iter__(self) -> Iterator[TaskT]:
+        """Lazily iterate `load()` with the config-layer system prompt applied and
+        any view transform on top — the read path; `load` is the subclass hook."""
+        prompt = self.system_prompt
+        tasks = (
+            task.with_system_prompt(prompt) if prompt is not None else task
+            for task in self.load()
+        )
+        yield from self.transform(tasks) if self.transform is not None else tasks
+
+    def view(self, transform: Callable[[Iterator[TaskT]], Iterator[TaskT]]) -> Self:
+        """A shallow copy of this taskset iterating through `transform`, composed
+        onto any transform this taskset already carries."""
+        clone = copy.copy(self)
+        prev = self.transform
+        clone.transform = (
+            transform if prev is None else lambda tasks: transform(prev(tasks))
+        )
+        return clone
+
+    def head(self, num_tasks: int) -> Self:
+        """A lazy, always-finite view of the first `num_tasks` tasks."""
+        view = self.view(lambda tasks: itertools.islice(tasks, num_tasks))
+        view.INFINITE = False
+        return view
+
+    def shuffle(self, seed: int | None = None) -> Self:
+        """A shuffled view under `seed` — the shared fixed seed when None, so runs
+        sample reproducibly (materializes the receiver on iteration); raises on an
+        infinite taskset — bound it first (`head(n).shuffle()`)."""
+        if self.INFINITE:
+            raise ValueError(
+                f"{type(self).__name__} is infinite - cannot shuffle; "
+                "bound it first with head(num_tasks)"
+            )
+
+        def shuffled(tasks: Iterator[TaskT]) -> Iterator[TaskT]:
+            materialized = list(tasks)
+            random.Random(SEED if seed is None else seed).shuffle(materialized)
+            return iter(materialized)
+
+        return self.view(shuffled)
+
+    @classmethod
+    def task_type(cls) -> type[Task]:
+        return concrete_type(cls, Task, origin=Taskset) or Task
+
+    def server_config(self, server_cls: type) -> BaseConfig:
+        """The config a `tools` entry is built with, resolved off `self.config` (the
+        taskset config; see `resolve_server_config`). Override to pair explicitly."""
+        return resolve_server_config(
+            type(self).__name__,
+            self.config,
+            server_cls,
+            sole=len(set(type(self).tools)) == 1,
+        )
+
+    def tool_servers(self) -> list[Toolset]:
+        return [cls(self.server_config(cls)) for cls in type(self).tools]

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/utils/generic.py

@@ -39,7 +39,7 @@ def deep_merge(base: dict, override: dict) -> dict:
     return merged
 
 
-def generic_type(
+def concrete_type(
     cls: type, bound: type[T], *, origin: type | None = None
 ) -> type[T] | None:
     """Find a concrete bounded type through `cls`'s MRO, most-derived first."""

{verifiers-0.2.2.dev50 → verifiers-0.2.2.dev52}/verifiers/v1/utils/sampling.py

@@ -1,10 +1,9 @@
 """Shared sampling: an optional fixed-seed shuffle, then an optional head-slice.
 
-Every entrypoint narrows its items the same way: with `--shuffle`, a shuffle under a fixed
-seed so the sampled subset is the *same* every run (reproducible), then an optional slice to
-the first `limit`. Taskset entrypoints (eval, validate, debug, GEPA) go through
-`Taskset.select`, which shares this shuffle; the server eval path and the legacy bridge
-sample plain index lists here directly.
+With `--shuffle`, a shuffle under the fixed `SEED` so the sampled subset is the *same*
+every run (reproducible), then an optional slice to the first `limit`. Used by the paths
+that sample plain index lists (the server eval path and the legacy bridge); tasksets
+shuffle themselves (`Taskset.shuffle`, same default seed).
 """
 
 import random