verifiers 0.2.2.dev49__py3-none-any.whl → 0.2.2.dev51__py3-none-any.whl

verifiers/v1/cli/debug.py

@@ -263,7 +263,14 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.select(config.num_tasks, config.shuffle)
+    if config.num_tasks is None and taskset.INFINITE:
+        raise ValueError(
+            f"{type(taskset).__name__} is infinite - bound the run with -n"
+        )
+    selected = taskset.shuffle() if config.shuffle else taskset
+    if config.num_tasks is not None:
+        selected = selected.head(config.num_tasks)
+    tasks = list(selected)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

verifiers/v1/cli/eval/runner.py

@@ -26,7 +26,15 @@ logger = logging.getLogger(__name__)
 async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     client = resolve_client(config.client)
-    tasks = env.taskset.select(config.num_tasks, config.shuffle)
+    taskset = env.taskset
+    if config.num_tasks is None and taskset.INFINITE:
+        raise ValueError(
+            f"{type(taskset).__name__} is infinite - bound the run with -n"
+        )
+    selected = taskset.shuffle() if config.shuffle else taskset
+    if config.num_tasks is not None:
+        selected = selected.head(config.num_tasks)
+    tasks = list(selected)
     ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     semaphore = (
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
@@ -132,9 +140,15 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
 
         # The client owns the taskset: load it here, once — the server (and its pool
         # workers) never load data, they rebuild each dispatched task from its request.
-        tasks = load_taskset(config.env.taskset).select(
-            config.num_tasks, config.shuffle
-        )
+        taskset = load_taskset(config.env.taskset)
+        if config.num_tasks is None and taskset.INFINITE:
+            raise ValueError(
+                f"{type(taskset).__name__} is infinite - bound the run with -n"
+            )
+        selected = taskset.shuffle() if config.shuffle else taskset
+        if config.num_tasks is not None:
+            selected = selected.head(config.num_tasks)
+        tasks = list(selected)
     # Spawned processes inherit no logging — hand them the main process's setup so
     # their rollout logs land in the output dir.
     level = "DEBUG" if config.verbose else "INFO"

verifiers/v1/cli/validate.py

@@ -203,7 +203,14 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.select(config.num_tasks, config.shuffle)
+    if config.num_tasks is None and taskset.INFINITE:
+        raise ValueError(
+            f"{type(taskset).__name__} is infinite - bound the run with -n"
+        )
+    selected = taskset.shuffle() if config.shuffle else taskset
+    if config.num_tasks is not None:
+        selected = selected.head(config.num_tasks)
+    tasks = list(selected)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):

verifiers/v1/configs/judge.py

@@ -5,7 +5,7 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
-from pydantic import BaseModel, SerializeAsAny, model_validator
+from pydantic import BaseModel, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
@@ -20,15 +20,8 @@ class JudgeConfig(BaseClientConfig):
     weight: float = 1.0
     model: str = "openai/gpt-5.4-nano"
     sampling: SamplingConfig = SamplingConfig()
-    prompt: str | None = None
-    prompt_file: Path | None = None
-    """Prompt file override, mutually exclusive with `prompt`."""
-
-    @model_validator(mode="after")
-    def check_prompt_source(self) -> "JudgeConfig":
-        if self.prompt is not None and self.prompt_file is not None:
-            raise ValueError("set `prompt` or `prompt_file`, not both")
-        return self
+    prompt: Path | None = None
+    """File whose text overrides the judge's default prompt template."""
 
 
 Judges = list[SerializeAsAny[JudgeConfig]]

verifiers/v1/configs/taskset.py

@@ -1,5 +1,7 @@
 """The taskset plugin's config: which rows load, under `--env.taskset.*`."""
 
+from pathlib import Path
+
 from pydantic import SerializeAsAny
 from pydantic_config import BaseConfig
 
@@ -14,6 +16,9 @@ class TasksetConfig(BaseConfig):
     positional `eval <taskset-id>`)."""
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
+    system_prompt: Path | None = None
+    """File whose text overrides each task's `TaskData.system_prompt` on
+    iteration (e.g. a GEPA `best_system_prompt.txt`)."""
 
     @property
     def name(self) -> str:

verifiers/v1/env.py

@@ -33,7 +33,7 @@ from verifiers.v1.retries import run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.task import Task, resolve_server_config
 from verifiers.v1.trace import Error, Trace
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
 
 logger = logging.getLogger(__name__)
@@ -83,7 +83,7 @@ class Env(ABC, Generic[ConfigT]):
     def __init__(self, config: ConfigT) -> None:
         from verifiers.v1.loaders import load_harness, load_taskset
 
-        config_cls = generic_type(type(self), EnvConfig, origin=Env) or EnvConfig
+        config_cls = concrete_type(type(self), EnvConfig, origin=Env) or EnvConfig
         if not isinstance(config, config_cls):
             raise TypeError(
                 f"{type(self).__name__} declares Env[{config_cls.__name__}], "

verifiers/v1/envs/agentic_judge/env.py

@@ -180,41 +180,31 @@ class JudgeTask(vf.Task):
 class JudgeTaskConfig(vf.BaseConfig):
     """The judge's minted task: the grading policy and what lands in its box."""
 
-    prompt: Path | str | None = None
-    """Grading-policy override: inline text, or a policy file (a value ending in
-    `.md`/`.txt` is read from disk). Replaces only the policy body — the verdict
-    contract and workspace note are always appended, so a custom policy cannot
-    break verdict scraping. May reference `{prompt}` (the solver task's prompt);
-    if it doesn't, the task statement is appended after the policy."""
-    hint: Path | str | None = None
-    """Optional hints injected as their own section (inline text or a `.md`/
-    `.txt` file): task-family pointers into the trace or box — e.g. for math,
-    where the reference answer lives in the record; for SWE, to diff the repo
-    or read `info.patch`."""
+    prompt: Path | None = None
+    """Grading-policy file. Replaces only the policy body — the verdict contract
+    and workspace note are always appended. May reference `{prompt}` (the solver
+    task's prompt); if it doesn't, the task statement is appended after."""
+    hint: Path | None = None
+    """Optional hints file injected as their own section: task-family pointers
+    into the trace or box — e.g. for math, where the reference answer lives in
+    the record; for SWE, to diff the repo or read `info.patch`."""
     rubric: Path | None = None
     """Criteria the judge grades against: a `.toml`/`.json` file with a
     `criteria` list — the plugged rubric judge's format, so the same rubric
     files work for both. None grades the single built-in `solved` criterion."""
 
-    @staticmethod
-    def _resolve(value: Path | str) -> str:
-        path = Path(value)
-        if isinstance(value, Path) or path.suffix in (".md", ".txt"):
-            return path.read_text(encoding="utf-8")
-        return str(value)
-
     def build_prompt(self) -> str:
         if self.prompt is None:
             return GRADE_PROMPT + "\n\n" + TASK_SECTION
-        return self._resolve(self.prompt)
+        return self.prompt.read_text()
 
     def build_hint(self) -> str | None:
-        return self._resolve(self.hint) if self.hint is not None else None
+        return self.hint.read_text() if self.hint is not None else None
 
     def criteria(self) -> list[Criterion]:
         if self.rubric is None:
             return [SOLVED]
-        text = self.rubric.read_text(encoding="utf-8")
+        text = self.rubric.read_text()
         data = (
             tomllib.loads(text)
             if self.rubric.suffix.lower() == ".toml"

verifiers/v1/gepa/adapter.py

@@ -68,14 +68,10 @@ class GEPAAdapter:
         )
 
     async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Episode]:
-        # Inject the candidate by rebuilding each Task around a data row with the new
-        # system_prompt (TaskData is frozen; behavior/config carry over unchanged).
-        tasks = [
-            type(t)(
-                t.data.model_copy(update={"system_prompt": system_prompt}), t.config
-            )
-            for t in (self.tasks[idx] for idx in batch)
-        ]
+        # Inject the candidate as a copy of each base task with its system_prompt overridden
+        # (`with_system_prompt` copies rather than reconstructs, so subclass state survives and
+        # the shared base task in `self.tasks` is left untouched for the next candidate).
+        tasks = [self.tasks[idx].with_system_prompt(system_prompt) for idx in batch]
         slots = [slot for task in tasks for slot in self.env.slots(task)]
         results = await asyncio.gather(
             *(

verifiers/v1/gepa/runner.py

@@ -37,7 +37,10 @@ class _GEPALog:
 
 def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
     logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
-    all_tasks = env.taskset.select(config.num_train + config.num_val, config.shuffle)
+    # Global shuffle: an infinite taskset raises here — run it with
+    # shuffle=false (there is no whole set to sample from).
+    taskset = env.taskset.shuffle() if config.shuffle else env.taskset
+    all_tasks = list(taskset.head(config.num_train + config.num_val))
     train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val)
     selected_tasks = [*train_tasks, *val_tasks]
     # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool —
@@ -105,7 +108,20 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
                 "skip_perfect_score": False,
                 "logger": _GEPALog(),
             }
-            return optimize(**optimize_kwargs)
+            result = optimize(**optimize_kwargs)
+            if run_dir is not None:
+                # Persist the winning prompt as a plain file so it can be handed straight to
+                # eval/train via `--env.taskset.system-prompt` (see TasksetConfig).
+                candidate = result.best_candidate
+                best = (
+                    candidate.get("system_prompt", "")
+                    if isinstance(candidate, dict)
+                    else str(candidate)
+                )
+                best_path = run_dir / "best_system_prompt.txt"
+                best_path.write_text(best, encoding="utf-8")
+                logger.info("best system prompt: %s", best_path)
+            return result
         finally:
             loop.run_until_complete(serving.__aexit__(None, None, None))
     finally:

verifiers/v1/judge.py

@@ -63,7 +63,7 @@ from verifiers.v1.configs.judge import (
 from verifiers.v1.dialects.chat import message_to_wire
 from verifiers.v1.scoring import parse_judge_choice
 from verifiers.v1.types import Messages, StrictBaseModel, Usage
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
     from verifiers.v1.task import TaskData
@@ -111,18 +111,18 @@ ConfigT = TypeVar("ConfigT", bound=JudgeConfig, default=JudgeConfig)
 
 def judge_config_cls(cls: type) -> type[JudgeConfig]:
     """Resolve a judge's config specialization through its MRO, else `JudgeConfig`."""
-    return generic_type(cls, JudgeConfig) or JudgeConfig
+    return concrete_type(cls, JudgeConfig) or JudgeConfig
 
 
 class Judge(Generic[ParsedT, ConfigT]):
     prompt: str | None = None
-    """Default prompt template, overridden by config."""
+    """Default prompt template, overridden by a config `prompt` file."""
     schema: type[BaseModel] | None = None
 
     def __init__(self, config: ConfigT | None = None) -> None:
         self.config = cast(ConfigT, config or judge_config_cls(type(self))())
-        if self.config.prompt_file is not None:
-            self.prompt = self.config.prompt_file.read_text(encoding="utf-8")
+        if self.config.prompt is not None:
+            self.prompt = self.config.prompt.read_text()
 
     @property
     def reward_name(self) -> str:
@@ -132,7 +132,7 @@ class Judge(Generic[ParsedT, ConfigT]):
         return judge_key(self.config) or fallback or "judge"
 
     def build_messages(self, **fields: Any) -> str | Messages:
-        template = self.config.prompt or self.prompt
+        template = self.prompt
         if template is None:
             raise ValueError(
                 f"{type(self).__name__} has no `prompt`; set it or override build_messages"

verifiers/v1/loaders.py

@@ -20,7 +20,7 @@ from verifiers.v1.harness import Harness
 from verifiers.v1.judge import Judge, judge_config_cls
 from verifiers.v1.task import Task
 from verifiers.v1.taskset import Taskset
-from verifiers.v1.utils.generic import generic_type, prefix_validation_error
+from verifiers.v1.utils.generic import concrete_type, prefix_validation_error
 from verifiers.v1.utils.install import ensure_installed
 
 
@@ -208,7 +208,7 @@ def load_judge(config: JudgeConfig) -> Judge:
 def taskset_config_type(taskset_id: str) -> type[TasksetConfig]:
     """Resolve the taskset's config specialization through its MRO."""
     return (
-        generic_type(taskset_class(taskset_id), TasksetConfig, origin=Taskset)
+        concrete_type(taskset_class(taskset_id), TasksetConfig, origin=Taskset)
         or TasksetConfig
     )
 
@@ -216,7 +216,7 @@ def taskset_config_type(taskset_id: str) -> type[TasksetConfig]:
 def harness_config_type(harness_id: str) -> type[HarnessConfig]:
     """Resolve the harness's config specialization through its MRO."""
     return (
-        generic_type(harness_class(harness_id), HarnessConfig, origin=Harness)
+        concrete_type(harness_class(harness_id), HarnessConfig, origin=Harness)
         or HarnessConfig
     )
 
@@ -231,7 +231,7 @@ def env_config_type(taskset_id: str, env_id: str = "") -> type[EnvConfig]:
     its MRO — `SingleAgentEnvConfig` for a plain taskset. The run's `env` field
     narrows to this, which is what gives `--env.<role>.model` addressing."""
     return (
-        generic_type(environment_class(taskset_id, env_id), EnvConfig, origin=Env)
+        concrete_type(environment_class(taskset_id, env_id), EnvConfig, origin=Env)
         or EnvConfig
     )
 

verifiers/v1/mcp/server.py

@@ -13,7 +13,7 @@ from pydantic import TypeAdapter, ValidationError
 from pydantic_config import BaseConfig
 
 from verifiers.v1.state import State, StateT, state_cls
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
     from httpx import AsyncClient, Response
@@ -286,7 +286,7 @@ class ServerBase(Generic[ConfigT, StateT]):
     @classmethod
     def _config_cls(cls) -> type[BaseConfig]:
         """Resolve the server's config specialization through its MRO."""
-        if config_cls := generic_type(cls, BaseConfig):
+        if config_cls := concrete_type(cls, BaseConfig):
             return config_cls
         raise TypeError(
             f"{cls.__name__} must parameterize its config, e.g. Toolset[MyConfig]"

verifiers/v1/state.py

@@ -8,7 +8,7 @@ from pydantic import ConfigDict
 from typing_extensions import TypeVar
 
 from verifiers.v1.types import StrictBaseModel
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 
 class State(StrictBaseModel):
@@ -20,4 +20,4 @@ StateT = TypeVar("StateT", bound=State, default=State)
 
 def state_cls(cls: type) -> type[State]:
     """Resolve a class's `State` specialization through its MRO, else `State`."""
-    return generic_type(cls, State) or State
+    return concrete_type(cls, State) or State

verifiers/v1/task.py

@@ -24,10 +24,11 @@ wraps it in the declared `Task` — one task type per taskset.
 
 from __future__ import annotations
 
+import copy
 import inspect
 import logging
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, ClassVar, Generic
+from typing import TYPE_CHECKING, ClassVar, Generic, Self
 
 from pydantic import ConfigDict, Field
 from pydantic_config import BaseConfig
@@ -38,7 +39,7 @@ from verifiers.v1.decorators import discover_decorated, invoke_all
 from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, StrictBaseModel, content_text
-from verifiers.v1.utils.generic import generic_type
+from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
     from verifiers.v1.judge import Judge
@@ -49,27 +50,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _requires_runtime(fn) -> bool:
-    param = inspect.signature(fn).parameters.get("runtime")
-    # A defaulted runtime parameter can still be called offline with None.
-    return param is not None and param.default is inspect.Parameter.empty
-
-
-def _record_result(
-    trace: Trace,
-    name: str,
-    result,
-    weight: float | None = None,
-) -> None:
-    """Record a scalar or keyed scoring result."""
-    values = list(result.items()) if isinstance(result, Mapping) else [(name, result)]
-    for key, value in values:
-        if weight is None:
-            trace.record_metric(key, value)
-        else:
-            trace.record_reward(key, value, weight)
-
-
 class TaskResources(StrictBaseModel):
     model_config = ConfigDict(frozen=True)
 
@@ -148,12 +128,12 @@ ConfigT = TypeVar("ConfigT", bound=TaskConfig, default=TaskConfig)
 
 def task_data_cls(cls: type) -> type[TaskData]:
     """Resolve a task's `TaskData` specialization through its MRO, else `TaskData`."""
-    return generic_type(cls, TaskData) or TaskData
+    return concrete_type(cls, TaskData) or TaskData
 
 
 def task_config_cls(cls: type) -> type[TaskConfig]:
     """Resolve a task's `TaskConfig` specialization through its MRO, else `TaskConfig`."""
-    return generic_type(cls, TaskConfig) or TaskConfig
+    return concrete_type(cls, TaskConfig) or TaskConfig
 
 
 def resolve_server_config(
@@ -197,6 +177,15 @@ class Task(Generic[DataT, StateT, ConfigT]):
         self.data = data
         self.config = config if config is not None else task_config_cls(type(self))()
 
+    def with_system_prompt(self, system_prompt: str) -> Self:
+        """A shallow copy of this task with `data.system_prompt` overridden. Copies the
+        instance instead of reconstructing via `type(self)(...)`, so a subclass with a
+        non-`(data, config)` constructor or extra load-time state keeps it. Used to apply the
+        config-layer / GEPA system prompt (see `TasksetConfig` and `verifiers.v1.gepa`)."""
+        clone = copy.copy(self)
+        clone.data = self.data.model_copy(update={"system_prompt": system_prompt})
+        return clone
+
     def plugged_judges(self) -> list[Judge]:
         from verifiers.v1.loaders import load_judge
 
@@ -229,6 +218,11 @@ class Task(Generic[DataT, StateT, ConfigT]):
         trace: Trace,
         runtime: Runtime | None = None,
     ) -> None:
+        def requires_runtime(fn) -> bool:
+            param = inspect.signature(fn).parameters.get("runtime")
+            # A defaulted runtime parameter can still be called offline with None.
+            return param is not None and param.default is inspect.Parameter.empty
+
         judges = self.plugged_judges()
         available = {"task": self.data, "trace": trace}
         if runtime is not None:
@@ -239,36 +233,50 @@ class Task(Generic[DataT, StateT, ConfigT]):
             rewards = discover_decorated(self, "reward")
             if runtime is None:
                 skipped = [
-                    fn.__name__ for fn in (*metrics, *rewards) if _requires_runtime(fn)
+                    fn.__name__ for fn in (*metrics, *rewards) if requires_runtime(fn)
                 ] + [
                     judge.reward_name
                     for judge in judges
-                    if _requires_runtime(judge.score)
+                    if requires_runtime(judge.score)
                 ]
                 if skipped:
                     logger.info(
                         "score: no runtime — skipped runtime-dependent signals: %s",
                         skipped,
                     )
-                metrics = [fn for fn in metrics if not _requires_runtime(fn)]
-                rewards = [fn for fn in rewards if not _requires_runtime(fn)]
+                metrics = [fn for fn in metrics if not requires_runtime(fn)]
+                rewards = [fn for fn in rewards if not requires_runtime(fn)]
                 judges = [
-                    judge for judge in judges if not _requires_runtime(judge.score)
+                    judge for judge in judges if not requires_runtime(judge.score)
                 ]
 
             metric_results = await invoke_all(metrics, available)
             for fn, result in zip(metrics, metric_results):
-                _record_result(trace, fn.__name__, result)
+                if isinstance(result, Mapping):
+                    trace.record_metrics(result)
+                else:
+                    trace.record_metric(fn.__name__, result)
             reward_results = await invoke_all(rewards, available)
             for fn, result in zip(rewards, reward_results):
-                _record_result(
-                    trace, fn.__name__, result, getattr(fn, "_vf_weight", 1.0)
+                weight = getattr(fn, "_vf_weight", 1.0)
+                items = (
+                    result.items()
+                    if isinstance(result, Mapping)
+                    else [(fn.__name__, result)]
                 )
+                for key, value in items:
+                    trace.record_reward(key, value, weight)
             judge_results = await invoke_all(
                 [judge.score for judge in judges], available
             )
             for judge, result in zip(judges, judge_results):
-                _record_result(trace, judge.reward_name, result, judge.config.weight)
+                items = (
+                    result.items()
+                    if isinstance(result, Mapping)
+                    else [(judge.reward_name, result)]
+                )
+                for key, value in items:
+                    trace.record_reward(key, value, judge.config.weight)
 
 
 TaskT = TypeVar("TaskT", bound=Task)

verifiers/v1/taskset.py

@@ -1,88 +1,106 @@
 """The taskset: a thin loader that yields typed tasks.
 
-A `Taskset` is the data half of an environment: config in, tasks out. `load()` — the
-one subclass hook — builds each row's `TaskData` and wraps it in the task type with
-the config's task-facing subtree:
+A `Taskset` is the data half of an environment: config in, tasks out. `load()` is
+the main hook that builds each task:
 
     def load(self) -> Iterable[MyTask]:
-        return [MyTask(MyData(idx=i, ...), self.config.task) for i in ...]
+        for i in ...:
+            yield MyTask(MyData(idx=i, ...), self.config.task)
 
-`load` may also be a generator, possibly infinite (declare `INFINITE = True`); runs
-materialize what they need through `select`, the env server pulls task by task.
-
-Load-time knobs live on the taskset config, task-facing knobs under its `task`
-subtree, shared tool servers on `tools`. All per-task behavior lives on the `Task`.
-
-The class stays generic (`Taskset[TaskT, TasksetConfigT]`) so the loaders can read
-the types: `taskset_config_type` narrows `--env.taskset.*` flags, `task_type` types
-the wire trace — one task type per taskset, so replay can rebuild saved rows.
+`load` may also be a generator for infinite tasksets. There is a one-to-one
+mapping between taskset and task type, i.e. a taskset may only yield one task
+type.
 """
 
 from __future__ import annotations
 
+import copy
 import itertools
-import logging
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, ClassVar, Generic
+import random
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterable, Iterator
+from typing import TYPE_CHECKING, ClassVar, Generic, Self
 
 from pydantic_config import BaseConfig
 from typing_extensions import TypeVar
 
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.task import Task, TaskT, resolve_server_config
-from verifiers.v1.utils.generic import generic_type
-from verifiers.v1.utils.sampling import sample
+from verifiers.v1.utils.generic import concrete_type
+from verifiers.v1.utils.sampling import SEED
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Toolset
 
-logger = logging.getLogger(__name__)
-
-
 TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetConfig)
 
 
-class Taskset(Generic[TaskT, TasksetConfigT]):
-    INFINITE: ClassVar[bool] = False
-    """Whether `load` yields tasks forever. Inherent to the taskset, not a config
-    knob: runs bound themselves with `select(num_tasks)`, and shuffle is impossible."""
+class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
+    INFINITE: bool = False
+    """Whether the taskset is infinite (yields tasks forever). Class-declared;
+    a `head(n)` view shadows it per instance (bounded by construction)."""
 
     tools: ClassVar[tuple[type[Toolset], ...]] = ()
-    """Tool server classes shared by one environment worker's rollouts."""
+    """Tool servers shared by all tasks in the taskset. The environment will
+    spawn a single, global instance, reused across tasks."""
 
     def __init__(self, config: TasksetConfigT) -> None:
         self.config = config
+        override = config.system_prompt
+        self.system_prompt = override.read_text() if override is not None else None
+        self.transform: Callable[[Iterator[TaskT]], Iterator[TaskT]] | None = None
+        """Iteration transform carried by `head`/`shuffle` views (see `view`)."""
+
+    @abstractmethod
+    def load(self) -> Iterable[TaskT]:
+        """Build and yield the taskset's tasks; may be a generator (see module doc)."""
+
+    def __iter__(self) -> Iterator[TaskT]:
+        """Lazily iterate `load()` with the config-layer system prompt applied and
+        any view transform on top — the read path; `load` is the subclass hook."""
+        prompt = self.system_prompt
+        tasks = (
+            task.with_system_prompt(prompt) if prompt is not None else task
+            for task in self.load()
+        )
+        yield from self.transform(tasks) if self.transform is not None else tasks
+
+    def view(self, transform: Callable[[Iterator[TaskT]], Iterator[TaskT]]) -> Self:
+        """A shallow copy of this taskset iterating through `transform`, composed
+        onto any transform this taskset already carries."""
+        clone = copy.copy(self)
+        prev = self.transform
+        clone.transform = (
+            transform if prev is None else lambda tasks: transform(prev(tasks))
+        )
+        return clone
+
+    def head(self, num_tasks: int) -> Self:
+        """A lazy, always-finite view of the first `num_tasks` tasks."""
+        view = self.view(lambda tasks: itertools.islice(tasks, num_tasks))
+        view.INFINITE = False
+        return view
+
+    def shuffle(self, seed: int | None = None) -> Self:
+        """A shuffled view under `seed` — the shared fixed seed when None, so runs
+        sample reproducibly (materializes the receiver on iteration); raises on an
+        infinite taskset — bound it first (`head(n).shuffle()`)."""
+        if self.INFINITE:
+            raise ValueError(
+                f"{type(self).__name__} is infinite - cannot shuffle; "
+                "bound it first with head(num_tasks)"
+            )
+
+        def shuffled(tasks: Iterator[TaskT]) -> Iterator[TaskT]:
+            materialized = list(tasks)
+            random.Random(SEED if seed is None else seed).shuffle(materialized)
+            return iter(materialized)
+
+        return self.view(shuffled)
 
     @classmethod
     def task_type(cls) -> type[Task]:
-        return generic_type(cls, Task, origin=Taskset) or Task
-
-    def load(self) -> Iterable[TaskT]:
-        raise NotImplementedError
-
-    def select(
-        self, num_tasks: int | None = None, shuffle: bool = False
-    ) -> list[TaskT]:
-        """Materialize the first `num_tasks` off `load` (all when `None`), pulled
-        lazily. `shuffle` samples from the whole taskset instead (fixed-seed), which
-        materializes everything first; on an `INFINITE` taskset it's a warned no-op —
-        the first `num_tasks` generated are already an arbitrary sample."""
-        if type(self).INFINITE:
-            if num_tasks is None:
-                raise ValueError(
-                    f"{type(self).__name__} is infinite - select a bounded subset "
-                    "with num_tasks (-n on the CLI)"
-                )
-            if shuffle:
-                logger.warning(
-                    "shuffle is a no-op on an infinite taskset - "
-                    "taking the first %d generated tasks",
-                    num_tasks,
-                )
-            return list(itertools.islice(self.load(), num_tasks))
-        if shuffle:
-            return sample(self.load(), shuffle=True, limit=num_tasks)
-        return list(itertools.islice(self.load(), num_tasks))
+        return concrete_type(cls, Task, origin=Taskset) or Task
 
     def server_config(self, server_cls: type) -> BaseConfig:
         """The config a `tools` entry is built with, resolved off `self.config` (the

verifiers/v1/tasksets/lean/taskset.py

@@ -61,7 +61,6 @@ class LeanTaskConfig(TaskConfig):
 class LeanConfig(TasksetConfig):
     dataset: LeanDatasetConfig
     docker_image: str = DEFAULT_DOCKER_IMAGE
-    system_prompt: str = DEFAULT_SYSTEM_PROMPT
     task: LeanTaskConfig = LeanTaskConfig()
 
 
@@ -187,7 +186,7 @@ class LeanTaskset(Taskset[LeanTask, LeanConfig]):
                     idx=index,
                     name=str(name) if name else f"task_{index:05d}",
                     prompt=self._build_prompt(formal_statement, header),
-                    system_prompt=config.system_prompt,
+                    system_prompt=DEFAULT_SYSTEM_PROMPT,
                     image=config.docker_image,
                     workdir=config.task.lean_project_path,
                     resources=resources,

verifiers/v1/utils/generic.py

@@ -39,7 +39,7 @@ def deep_merge(base: dict, override: dict) -> dict:
     return merged
 
 
-def generic_type(
+def concrete_type(
     cls: type, bound: type[T], *, origin: type | None = None
 ) -> type[T] | None:
     """Find a concrete bounded type through `cls`'s MRO, most-derived first."""

verifiers/v1/utils/sampling.py

@@ -1,10 +1,9 @@
 """Shared sampling: an optional fixed-seed shuffle, then an optional head-slice.
 
-Every entrypoint narrows its items the same way: with `--shuffle`, a shuffle under a fixed
-seed so the sampled subset is the *same* every run (reproducible), then an optional slice to
-the first `limit`. Taskset entrypoints (eval, validate, debug, GEPA) go through
-`Taskset.select`, which shares this shuffle; the server eval path and the legacy bridge
-sample plain index lists here directly.
+With `--shuffle`, a shuffle under the fixed `SEED` so the sampled subset is the *same*
+every run (reproducible), then an optional slice to the first `limit`. Used by the paths
+that sample plain index lists (the server eval path and the legacy bridge); tasksets
+shuffle themselves (`Taskset.shuffle`, same default seed).
 """
 
 import random

{verifiers-0.2.2.dev49.dist-info → verifiers-0.2.2.dev51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev49
+Version: 0.2.2.dev51
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev49.dist-info → verifiers-0.2.2.dev51.dist-info}/RECORD

@@ -170,35 +170,35 @@ verifiers/utils/version_utils.py,sha256=39gV8-mNF9GspAgKloLo8oy7_MVluMrwGPrntQv2
 verifiers/v1/__init__.py,sha256=xHKE-HZA8bN0JJovf4KbBI2mxBtZJ4hPNnzac4FtWwM,7332
 verifiers/v1/agent.py,sha256=dj4GabQSs6Wzwbph5Rlx-efsVQsSuWV2YI5hfvq5mGA,31990
 verifiers/v1/decorators.py,sha256=9CcCwgZlRjVGPqT5G1CFJUHP4OIMuBGkUJZkcI6dnbk,3865
-verifiers/v1/env.py,sha256=3R3O_2dEk0feql_HKbvzlVqvRdwSjHfWML1aEJ0rZKY,18445
+verifiers/v1/env.py,sha256=m-1TNzs9YyFbMkoW6c54vI5xG4jnW3LvLgvJhOgbg04,18447
 verifiers/v1/episode.py,sha256=KWyM9ovTphEIo2260BYu2HvtnOOTxjLRDPrF_jTCiuA,2497
 verifiers/v1/errors.py,sha256=slZnrtqMo_BgXQV46wJJhV6vvAAnVsuCbVGp2vBVWfs,6888
 verifiers/v1/graph.py,sha256=-j8MNdIv1r5ifajv5ggnGPsfBNB74IO3WKLcrpYh9hg,29102
 verifiers/v1/harness.py,sha256=cDiDuhp29aqL4CxiqSZz-IT0QlOcVlye_vTTHZblWo8,10948
-verifiers/v1/judge.py,sha256=VnTi9iSfCJMCccYWc1qCTcT7iD3AcomS2cXfHwh1HVw,9423
+verifiers/v1/judge.py,sha256=duZcWYTynzqShs8vfbExpFlgLS-JA9NFqdmCRLg3sbc,9393
 verifiers/v1/legacy.py,sha256=nkbdhFJZN1QzsjE42AB_A_c1zrk3_Y8zR07srZJS2ck,22584
-verifiers/v1/loaders.py,sha256=TnNKY2msVo3p7m-5wjw376PYH-zTu3TLixXRst0VOiM,9985
+verifiers/v1/loaders.py,sha256=NSIBdNU3JVeyddL-dM4dJt6khCHyjCgzJuFtLrgiwD0,9989
 verifiers/v1/push.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtvI,11363
 verifiers/v1/retries.py,sha256=quUGVruUkZx-9WQcFiocMfL_M056voU8gwDqsuLR3f4,5412
 verifiers/v1/rollout.py,sha256=GJzHgJFlw8O1vDtWM__2JhSiG2GqbfjtH2ri1MRfw3g,20495
 verifiers/v1/scoring.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/session.py,sha256=J6ZPuvsg6vMTJAs9dMUMtfdjCuvp6Hom877DVdKFKlY,6912
-verifiers/v1/state.py,sha256=R8tyQv8nsFV2pztrquDqCOa1Mk1fAp3w2GjqUugZwE0,689
-verifiers/v1/task.py,sha256=ad2YgS7ZeeBdezyWpuexLeGsCyIk5uuUJ061LXrYJ88,11074
-verifiers/v1/taskset.py,sha256=PX1-skAVhSpGF07qjxcG0Ctmq1LiMftMxxok-uc8qRY,3939
+verifiers/v1/state.py,sha256=LhV7lpqcC19_ZQZ4KKO4dzTmZHJO4VEOpjPaLMBWh54,691
+verifiers/v1/task.py,sha256=hRDIHAdBwP-aalLmhHeJ1VWW_9YJAkP21w7TkOU7VXU,11805
+verifiers/v1/taskset.py,sha256=RPFkpRkeEyPX557_mK-JwraBuUKsguiCUKSFyotTclU,4636
 verifiers/v1/trace.py,sha256=9KqBeGaRUtWh6Gpdf6G1c7UZso0DB9Mjmtkzl6I_oYE,19458
 verifiers/v1/types.py,sha256=5tZyG4r4bLJ9a14oybHy7bSCpt5X5A17eRwmIszuRo8,9118
 verifiers/v1/acp/__init__.py,sha256=9dwH6fLopNndRmcpRSYC8ghzzMquaLfgh645QM-Dwic,2189
 verifiers/v1/acp/_runner.py,sha256=FqEnHR0Q_YrCt_EuaJqiqTpd2_kFa41FqcEunwOhvTI,6856
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-verifiers/v1/cli/debug.py,sha256=JZxsrUZyHpuEDiZQeIbHpMhhh6JtfOB8khPhzprQAYg,11223
+verifiers/v1/cli/debug.py,sha256=mRBPlYjc93o-4sooVSJXl_H3zALC2vhAl3EBzN-DpoM,11507
 verifiers/v1/cli/gepa.py,sha256=hLIbS_4jBx0vvy4PXoePR5riFXcNvzQCMixZ-g2OR_U,4026
 verifiers/v1/cli/init.py,sha256=dVpX1YMz7DFmHgskAsPjP-HmVTGoZfC0ZUyE-T4I6l0,8322
 verifiers/v1/cli/output.py,sha256=pMS5Z0EBgyAVShNCAk9jVURNeLOaTfeRWpt0J8hHjq8,5890
 verifiers/v1/cli/replay.py,sha256=Jjyi61fl20xjGArdXTwZ3qKoSkC0iQ6WLSrvKkURdeI,10179
 verifiers/v1/cli/resolve.py,sha256=OsNw4C3r9xOVFa-mminJFcex_bAsJ6EQ8K7A_H8u2kQ,4370
 verifiers/v1/cli/serve.py,sha256=_mdJaHVVe4pXzcpAot0ugcmiN3ra2gS4GrGmLghnX5M,2866
-verifiers/v1/cli/validate.py,sha256=s6M1RXqfnhoe1IfPW25I6aWjFpVK0kZ5YSMdDOtjcOo,9976
+verifiers/v1/cli/validate.py,sha256=lZ_wWnLgIWH5rUlMtwzvF3uS_cFZA4ZqBowpzdI4gos,10260
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
 verifiers/v1/cli/dashboard/eval.py,sha256=i_Qn9enCU7aas-VTVAJocPNbTuHSUHThwQnls-3_kc4,34265
@@ -207,7 +207,7 @@ verifiers/v1/cli/dashboard/validate.py,sha256=rLsQ_31DjTIZpC_VO7zSG41ncUEBxqCx8m
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/eval/main.py,sha256=zvbLy_ryn3S1fWoTmWL4QEReVO1Q_hIrgyK6Fp0q_ZM,5501
 verifiers/v1/cli/eval/resume.py,sha256=QwXuLPs2lN-aMZk1CyaT12egRPzeI6vfY39DAd0nYLU,7171
-verifiers/v1/cli/eval/runner.py,sha256=wEOGgYh9l_BAXQKviCjmCvEofFkmFPf7T3lbZKj4Iiw,11508
+verifiers/v1/cli/eval/runner.py,sha256=TKBnHSSs_knZcaHiH0ZjrF8DneEWlLp5b8O_SjQz_as,12130
 verifiers/v1/clients/__init__.py,sha256=bHGS5jfrAZ-VBUeXpasXkkKdHkFgjOSN9KbOJXrnFtU,511
 verifiers/v1/clients/client.py,sha256=9gZScQM6CVQq9RZ_3slVBEsik6tr56uOFSVNvQfMdFQ,3288
 verifiers/v1/clients/config.py,sha256=QMR0PkpFLcLi_Zw6bVimigNjNiWeJ-4YvJ_JsSS9yGw,5228
@@ -217,12 +217,12 @@ verifiers/v1/configs/__init__.py,sha256=X7u6X7B3ieD1HZl79Tv1kg-yPHbT0QLvAebaGqxV
 verifiers/v1/configs/agent.py,sha256=N2Lm0iZXFxfsSIiXUQ7f3RDZcoin3M7pJj6HVdCE5eo,3537
 verifiers/v1/configs/env.py,sha256=4aUap3BJX3WCA2_fZ-jGPxF3khiVpORqmz_3eBT4j9w,7824
 verifiers/v1/configs/harness.py,sha256=CG8hGbHl3rr6yawH3YYMAFl01M7Piiwf4w5oiw2Vagg,1564
-verifiers/v1/configs/judge.py,sha256=ZJlmWogHY8oDjvWvtNuVKvkzuheWalIjl1njJ4dE0Mk,2511
+verifiers/v1/configs/judge.py,sha256=eEVtBGjAOmm1sSRd_UHVBpJEfUxMHxHDBFnvchmxs-w,2217
 verifiers/v1/configs/legacy.py,sha256=YOJM6L5u6-jYpZ1CLpPInufKdmz1lEsMNKIRdT2JaaY,1080
 verifiers/v1/configs/retries.py,sha256=-nPlgmz7J_NZ4GOYG6oCc3thXo4La2SufBmvQGvvPGo,821
 verifiers/v1/configs/serve.py,sha256=cHHnFGQtqEwvaovdeoVMpCebTHgPXKUuIivN5IhxZR0,2596
 verifiers/v1/configs/task.py,sha256=1ozZjfpahin7I_TqzC7f3xr7CHSthAdydegFPXW4d9k,1101
-verifiers/v1/configs/taskset.py,sha256=IdZwsarn0_WvYdR_Kr4QYYdqbaoGYkbYyO5VA-RPYPc,657
+verifiers/v1/configs/taskset.py,sha256=o-tr1xkmGmYG10_ySG-Du_djWS-dN4h7cv14WSvJ4eY,851
 verifiers/v1/configs/cli/__init__.py,sha256=3wBAONw5XkBPqbMjpHk5XBHlBVybR8JQ2JCPWijYBMU,444
 verifiers/v1/configs/cli/debug.py,sha256=3lkKqmQFLJBcUZECChBHCrod5pbCKSmT47EhjILrxjI,2843
 verifiers/v1/configs/cli/env.py,sha256=XgjRXYpPjga0ZYIHEESY7VuVvtE6pWfAzf6JBLeuTIc,2695
@@ -238,7 +238,7 @@ verifiers/v1/dialects/chat.py,sha256=R_otqi5N0snB1QVKIYloNeLHR9kc7OmI06HYg1ir_Lw
 verifiers/v1/dialects/responses.py,sha256=ismnxUikPcPUIm80FUTPBbj-Cn1e0yGAsISMEseAWRc,13679
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=X7vQbbbfmJ_j-mJEfBkuB4a8QUPkUYkpG7yp0xYOEmg,279
-verifiers/v1/envs/agentic_judge/env.py,sha256=3XhGiMdKJBhpSyUdlfYYKPIR42Ll34WUvEFqbCtKdX8,15342
+verifiers/v1/envs/agentic_judge/env.py,sha256=2g9yz3QfYUc_OZBkxzWOEKsGiT51siYMKB7PwvbCY2E,14878
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
@@ -246,11 +246,11 @@ verifiers/v1/envs/single_agent/env.py,sha256=lPs_a6VA7KoH1xqA3T8S32Jzb_d4zrx0_54
 verifiers/v1/envs/user_sim/__init__.py,sha256=nnT695HdqjAEouYL5wtVu6MhIp2wf5yuK-EYgxOVx0s,118
 verifiers/v1/envs/user_sim/env.py,sha256=9L_KXjzfkTnDeQpn4ftQagZyPkJ3yjcQeZqZaq7kxjE,4567
 verifiers/v1/gepa/__init__.py,sha256=6nmdRE0-34AKioPHBjTxUg5Jo_2z7tMX-OU3zMNpAJI,197
-verifiers/v1/gepa/adapter.py,sha256=my7hM6GSzOw__F7Ot1E-XA_OhZyovhgbJ7fOQkW66l8,6149
+verifiers/v1/gepa/adapter.py,sha256=YNvHMR2L5Utl-vtPfjZxmDCd2tTcoAPWG6aVGnV1icM,6139
 verifiers/v1/gepa/config.py,sha256=gFn8Re4GoeHJs2PZ-iw4jCEZ72AiM1lNgwaA0uvJlYw,4626
 verifiers/v1/gepa/dataset.py,sha256=N2QYqUljHVuAdCbUuydEIlarvAWa6LFdGKu9qsOvjro,2186
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
-verifiers/v1/gepa/runner.py,sha256=o7Am2Ram-Ejbr_tVpN70zb0uQ5a1NIpXvlMhwWL22Xs,5406
+verifiers/v1/gepa/runner.py,sha256=-LZEt0lod6NKfyxg2dvOXZjfzTv3AWwendexd1De26c,6260
 verifiers/v1/harnesses/__init__.py,sha256=3fgfAMzFrNgu7RIO_evZPxipPdiskiHun5EKsdS4y6k,1301
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
 verifiers/v1/harnesses/bash/harness.py,sha256=ZtjhsANuK3vXkf9mRCi8-sBMyPebJMBNHa58vTd2Fm8,5284
@@ -291,7 +291,7 @@ verifiers/v1/judges/rubric.py,sha256=kB0yWEtfC60zJBCVjOKBO00YtH8-AOofzPkuyccVTtQ
 verifiers/v1/judges/rubric.txt,sha256=3KA2ZeYYdcDweJhUGopxu9EFgARhu2aWft5S5cj8Bc8,323
 verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,408
 verifiers/v1/mcp/launch.py,sha256=lRtcl98BO7imC44bsxO2yljwRQR_KCvJ6P-bd_hS5L8,18444
-verifiers/v1/mcp/server.py,sha256=uW8qddXQbtECxIPCU-lkOlFIv3VKWoFAeZFNhwPoyWo,10985
+verifiers/v1/mcp/server.py,sha256=zS3-m-0IX3z8nVzUhQj391f8_OieE1nKr5S4Nj0dSSU,10987
 verifiers/v1/mcp/toolset.py,sha256=tSPXgTNREZyaeCnoFVnDfXxFAj_pStZMPUIgyPrY_Dk,996
 verifiers/v1/runtimes/__init__.py,sha256=TqNtJVahzQuLS4fgCVeT9xdOzTkp0hRAPOkl7eFVDUg,2011
 verifiers/v1/runtimes/base.py,sha256=sTCiLo51yAU_T-Voc3BTH-FuzyEM0cGY05VzFE4DmYc,14720
@@ -311,7 +311,7 @@ verifiers/v1/tasksets/harbor/__init__.py,sha256=wwVGA8N7E2Q2qA5WGHuLRrn4DgG5oaRH
 verifiers/v1/tasksets/harbor/taskset.py,sha256=2Iw1ZaPRhRjdbfClRLd0iqKWABjM7M525zFN4wuaeCk,13867
 verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
 verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
-verifiers/v1/tasksets/lean/taskset.py,sha256=0KujQgx2fTL_DyWSQidebKoD94U8oJp82PXoco6-3B4,9065
+verifiers/v1/tasksets/lean/taskset.py,sha256=Fcn4UgTcdjMYnJ4CThaFZ28Rz_QDx0e21vJdW_vp12g,9019
 verifiers/v1/tasksets/openenv/__init__.py,sha256=G726meYYDplC3QE3AyjdSShq7FUcb4NkVqowuwPPuJk,303
 verifiers/v1/tasksets/openenv/taskset.py,sha256=1Wc9-5eN2HMKAwL34yTNex3zYOvXNXibF5wCROoMbFI,6075
 verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2LtcI7VDUHYPzRE,329
@@ -320,17 +320,17 @@ verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
 verifiers/v1/utils/compile.py,sha256=plpZYt-UD8q7Yx5pD7Rfy1w7Jer5TCBk2CJGHiNcPio,5696
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
-verifiers/v1/utils/generic.py,sha256=xwu8xNGdVOhM2K87C5cqhmnRf6apSPDASA15M0S2MaY,2054
+verifiers/v1/utils/generic.py,sha256=2VIN9RapMO9pVkFHB1UKUHck1vlTJ1pjfad2NzxuuXA,2055
 verifiers/v1/utils/git.py,sha256=Mj2InIZlCAKWEWR9HPEMJX1OgkCVve5NjWgTCD4GmkM,4871
 verifiers/v1/utils/image.py,sha256=OFw_wdwVtbdwa8uJ3X3vYfhpUAi0CTH5HMBupjVsRRQ,282
 verifiers/v1/utils/install.py,sha256=fWNsyKrw_PyhC0Qhqv5Ri0adFm5Ddx4AVy_hbsra1GU,1390
 verifiers/v1/utils/interrupt.py,sha256=F-KKhc5ndPJJfhd3SuMqyqhXhA32FhCRy5KWFJpEoM4,1179
 verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y,2084
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
-verifiers/v1/utils/sampling.py,sha256=JczGzBn6s3wsIrSY2Hy3hE8m6NreNgsNzhSjDikQqX0,1037
+verifiers/v1/utils/sampling.py,sha256=52TJ5-Hmpvp4YSgKNR-LmAPVwm4RDcH-o2-BnbVTmYQ,965
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev49.dist-info/METADATA,sha256=58Q74xtCzA2vwqY2VjgOgFdSvyqzrV2vAyCCkgjOrCo,4545
-verifiers-0.2.2.dev49.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev49.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev49.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev49.dist-info/RECORD,,
+verifiers-0.2.2.dev51.dist-info/METADATA,sha256=Ns-l7u4GlaZ2rco5UvfL-YtVj50LqCMzWiFX-zmXDEA,4545
+verifiers-0.2.2.dev51.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev51.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev51.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev51.dist-info/RECORD,,