verifiers 0.2.1.dev6__py3-none-any.whl → 0.2.1.dev7__py3-none-any.whl

verifiers/v1/cli/debug.py

@@ -3,9 +3,7 @@
 import asyncio
 import contextlib
 import logging
-import random
 import shlex
-import signal
 import sys
 import time
 import traceback
@@ -30,7 +28,9 @@ from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace, TraceTask
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -267,11 +267,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.load()
-    if config.shuffle:
-        random.Random(0).shuffle(tasks)
-    if config.num_tasks is not None:
-        tasks = tasks[: config.num_tasks]
+    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):
@@ -327,8 +323,9 @@ def main(argv: list[str] | None = None) -> None:
     sys.argv = [sys.argv[0], *argv]
     config = cli(config_type)
     setup_logging("DEBUG" if config.verbose else "INFO")
-    # Translate SIGTERM so async finally blocks still tear down their resources.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # Graceful shutdown: first Ctrl-C/SIGTERM unwinds each task's teardown `finally`;
+    # a second is swallowed so it can't orphan containers/sandboxes mid-cleanup.
+    install_interrupt()
     asyncio.run(run_debug(config))
 
 

verifiers/v1/cli/eval/main.py

@@ -7,7 +7,7 @@ import sys
 from pydantic_config import cli
 
 import verifiers.v1 as vf
-from verifiers.v1.utils.interrupt import install as install_interrupt
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 from verifiers.v1.cli.output import output_path, write_config
 from verifiers.v1.cli.resolve import (

verifiers/v1/cli/eval/runner.py

@@ -3,7 +3,6 @@
 import asyncio
 import contextlib
 import logging
-import random
 import time
 
 from verifiers.v1.clients import ModelContext, resolve_client
@@ -14,21 +13,15 @@ from verifiers.v1.cli.output import append_trace, output_path, save_config
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.env import Environment
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
-_SHUFFLE_SEED = (
-    0  # fixed so `--shuffle` samples the same tasks every run (reproducible)
-)
-
 
 async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     client = resolve_client(config.client)
-    tasks = env.taskset.load()
-    if config.shuffle:
-        random.Random(_SHUFFLE_SEED).shuffle(tasks)
-    tasks = tasks if config.num_tasks is None else tasks[: config.num_tasks]
+    tasks = sample(env.taskset.load(), config.shuffle, config.num_tasks)
     ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     # One episode of `num_rollouts` rollouts per task; the shared semaphore bounds total
     # concurrent rollouts (across episodes), so group rewards still see their whole episode.
@@ -169,11 +162,7 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
         await client.wait_for_server_startup(timeout=600)
         info = await client.info()
         group_scored = info.requires_group_scoring
-        idxs = list(range(info.num_tasks))
-        if config.shuffle:
-            random.Random(_SHUFFLE_SEED).shuffle(idxs)
-        if config.num_tasks is not None:
-            idxs = idxs[: config.num_tasks]
+        idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
         out = output_path(config)
         finished: list[Trace] = []
         if config.resume is not None:

verifiers/v1/cli/gepa.py

@@ -0,0 +1,74 @@
+"""The GEPA entrypoint: `uv run gepa [<taskset-id>] --model <model> [options]`.
+
+Registered as the `gepa` console script. Optimizes a v1 taskset's `Task.system_prompt` via
+GEPA (Genetic-Pareto): alternating rollouts with a teacher LM reflecting on results — see
+`verifiers.v1.gepa`. CLI resolution mirrors `eval`/`serve` (`verifiers.v1.cli.resolve`): a
+leading bare token is the taskset id, the taskset/harness subconfigs are narrowed from their
+`id`s so `--taskset.*` / `--harness.*` stay typed and `-h` renders them, `@ file.toml` loads,
+and the actual parse is `pydantic_config.cli`. v1-native tasksets only — a legacy (v0) env is
+rejected; run those through the existing `vf-gepa` command instead.
+"""
+
+import logging
+import sys
+
+from pydantic_config import cli
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.output import output_path, write_config
+from verifiers.v1.cli.resolve import (
+    extract_id,
+    narrow_config,
+    references_config_file,
+    with_positional_taskset,
+)
+from verifiers.v1.gepa import GEPAConfig, run_gepa
+from verifiers.v1.utils.interrupt import install_interrupt
+from verifiers.v1.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+USAGE = "usage: uv run gepa [<taskset-id>] [--harness.id <id>] --model <model> [options] [@ file.toml]"
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
+
+    if not argv or any(arg in ("-h", "--help") for arg in argv):
+        print(USAGE)
+        sys.argv = [sys.argv[0], "--help"]
+        cli(
+            narrow_config(GEPAConfig, argv)
+        )  # full option help, narrowed to the given ids
+        return
+    if not extract_id(argv, "taskset") and not references_config_file(argv):
+        raise SystemExit(
+            USAGE
+        )  # need a taskset (positional / --taskset.id) or a @ file.toml
+
+    config_type = narrow_config(GEPAConfig, argv)
+    sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
+    config = cli(config_type)
+    if config.is_legacy:
+        raise SystemExit(
+            "gepa optimizes native v1 tasksets; run a legacy (v0) environment through "
+            "`vf-gepa` instead of `gepa`."
+        )
+    setup_logging("DEBUG" if config.verbose else "INFO")
+    if config.dry_run:  # resolved + validated; write it to the output dir and exit
+        logger.info("wrote config to %s", write_config(config, output_path(config)))
+        return
+
+    # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out run still
+    # runs the runner's `serving()` teardown (interception pool / tool-server runtimes); further
+    # signals during that cleanup are swallowed so an impatient second Ctrl-C can't orphan them.
+    install_interrupt()
+
+    env = vf.Environment(config)
+    try:
+        result = run_gepa(env, config)
+    except KeyboardInterrupt:
+        # Graceful cleanup already ran (run_gepa's finally tore down serving); exit on the
+        # conventional Ctrl-C code without dumping a traceback.
+        raise SystemExit(130)
+    print(f"best system prompt:\n{result.best_candidate.get('system_prompt', '')}")

verifiers/v1/cli/replay.py

@@ -10,7 +10,6 @@ config is the base for replay-specific overrides.
 import asyncio
 import contextlib
 import logging
-import signal
 import sys
 import time
 import tomllib
@@ -31,6 +30,7 @@ from verifiers.v1.configs.replay import ReplayConfig
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task, WireTaskData, task_data_cls
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
@@ -220,8 +220,9 @@ def main(argv: list[str] | None = None) -> None:
         logging.lastResort = None
     else:
         setup_logging(level, log_file=log_file, console=True)
-    # Translate SIGTERM so async finally blocks still tear down their resources.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # Graceful shutdown: first Ctrl-C/SIGTERM unwinds the scoring teardown `finally`;
+    # a second is swallowed so it can't orphan resources mid-cleanup.
+    install_interrupt()
     asyncio.run(run_replay(config, source, out))
 
 

verifiers/v1/cli/validate.py

@@ -3,8 +3,6 @@
 import asyncio
 import contextlib
 import logging
-import random
-import signal
 import sys
 import time
 from typing import Any
@@ -26,7 +24,9 @@ from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -198,11 +198,7 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.load()
-    if config.shuffle:
-        random.Random(0).shuffle(tasks)
-    if config.num_tasks is not None:
-        tasks = tasks[: config.num_tasks]
+    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):
@@ -273,9 +269,9 @@ def main(argv: list[str] | None = None) -> None:
     setup_logging("DEBUG" if config.verbose else "INFO", console=not config.rich)
     if config.rich:
         logging.lastResort = None  # drop stdlib records that bypass loguru
-    # Make SIGTERM behave like Ctrl-C so a killed run still runs each task's `finally`
-    # (tears down containers/sandboxes) — and the atexit backstop catches the rest.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # Graceful shutdown: first Ctrl-C/SIGTERM unwinds each task's teardown `finally`
+    # (containers/sandboxes); a second is swallowed so it can't orphan them mid-cleanup.
+    install_interrupt()
     asyncio.run(run_validate(config))
 
 

verifiers/v1/gepa/__init__.py

@@ -0,0 +1,12 @@
+"""GEPA (Genetic-Pareto) system-prompt optimization for native v1 environments.
+
+`run_gepa(env, config)` drives the third-party `gepa` optimizer against a v1 `Environment`
+via `GEPAAdapter`, seeding from and improving `Task.system_prompt`. The `gepa` console
+script (`verifiers.v1.cli.gepa`) is a thin entrypoint over this package.
+"""
+
+from verifiers.v1.gepa.adapter import GEPAAdapter
+from verifiers.v1.gepa.config import GEPAConfig
+from verifiers.v1.gepa.runner import run_gepa
+
+__all__ = ["GEPAAdapter", "GEPAConfig", "run_gepa"]

verifiers/v1/gepa/adapter.py

@@ -0,0 +1,112 @@
+"""The GEPA <-> v1 bridge: run a candidate system prompt over a batch of tasks and score it.
+
+GEPA's adapter protocol (`evaluate`, `make_reflective_dataset`) is synchronous and
+`gepa.api.optimize` blocks, but v1 rollouts are async. The runner manages one event loop by
+hand: it enters `env.serving()` on that loop and runs the blocking `optimize()` on the main
+thread, so each synchronous `evaluate()` drives its batch of rollouts with
+`loop.run_until_complete` — the one sync↔async hop. (Mirrors how v0 vf-gepa bridged; no worker
+thread, so a Ctrl-C unwinds straight through `optimize()` into the runner's teardown.)
+"""
+
+import asyncio
+from collections.abc import Awaitable
+from dataclasses import dataclass, field
+from typing import Any, Callable, Mapping, Sequence
+
+from gepa.core.adapter import EvaluationBatch
+from pydantic_core import to_jsonable_python
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.env import Environment
+from verifiers.v1.task import Task
+from verifiers.v1.trace import Trace
+
+Candidate = dict[str, str]
+
+
+@dataclass
+class GEPAAdapter:
+    """Bridges GEPA's optimization loop with a native v1 `Environment`. `tasks` covers only the
+    trainset + valset tasks GEPA was given (not the whole taskset), keyed by `task.data.idx` —
+    GEPA's `batch` is a list of those idxs, and injecting the candidate rebuilds each `Task`
+    around a `data` row carrying the new `system_prompt`. `loop` is the runner's persistent event
+    loop (which holds `env.serving()` open); `evaluate` drives its rollouts on it with
+    `run_until_complete`."""
+
+    env: Environment
+    ctx: ModelContext
+    tasks: dict[int, Task]
+    loop: asyncio.AbstractEventLoop
+    semaphore: asyncio.Semaphore | None = None
+    on_complete: Callable[[Trace], Awaitable[None]] | None = None
+    """Called with each rollout's trace as it finalizes — the runner's persist hook that
+    streams traces to `traces.jsonl`, exactly as `run_eval` does."""
+    reflection_columns: list[str] = field(default_factory=list)
+    propose_new_texts: Callable[..., Candidate] | None = None
+    """Part of GEPA's adapter protocol — its proposer reads this attribute on every reflection
+    step. None = use GEPA's default reflection-LM proposer (the AttributeError from leaving it
+    undeclared silently disables all mutation proposals)."""
+
+    def evaluate(
+        self,
+        batch: list[int],
+        candidate: Candidate,
+        capture_traces: bool = False,
+    ) -> EvaluationBatch[Trace, Trace]:
+        """Run `candidate`'s system prompt on the tasks named by `batch` (`Task.idx` values)
+        and score them. Called synchronously by GEPA on the main thread; each batch's rollouts
+        run on the runner's persistent loop via `run_until_complete`."""
+        system_prompt = candidate.get("system_prompt", "")
+        traces = self.loop.run_until_complete(self._run_batch(batch, system_prompt))
+        scores = [trace.reward for trace in traces]
+        return EvaluationBatch(
+            outputs=traces,
+            scores=scores,
+            trajectories=traces if capture_traces else None,
+        )
+
+    async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Trace]:
+        # Inject the candidate by rebuilding each Task around a data row with the new
+        # system_prompt (TaskData is frozen; behavior/config carry over unchanged).
+        tasks = [
+            type(t)(
+                t.data.model_copy(update={"system_prompt": system_prompt}), t.config
+            )
+            for t in (self.tasks[idx] for idx in batch)
+        ]
+        episodes = [self.env.episode(task, self.ctx, n=1) for task in tasks]
+        results = await asyncio.gather(
+            *(episode.run(self.semaphore, self.on_complete) for episode in episodes)
+        )
+        return [trace for episode_traces in results for trace in episode_traces]
+
+    def make_reflective_dataset(
+        self,
+        candidate: Candidate,  # noqa: ARG002 - required by GEPA's adapter protocol
+        eval_batch: EvaluationBatch[Trace, Trace],
+        components_to_update: list[str],
+    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
+        """Build the reflective dataset the teacher LM reads to propose a new system prompt,
+        from `eval_batch.trajectories` (traces captured by a prior `evaluate(capture_traces=True)`
+        on the same batch)."""
+        traces = eval_batch.trajectories or []
+        records = []
+        for trace, score in zip(traces, eval_batch.scores):
+            record: dict[str, Any] = {
+                "query": trace.task.data.prompt_text,
+                "completion": trace.last_reply,
+                "reward": score,
+            }
+            if trace.has_error:
+                record["error"] = str(trace.error)
+            if trace.stop_condition:
+                record["stop_condition"] = trace.stop_condition
+            for column in self.reflection_columns:
+                if column in trace.info:
+                    record[column] = to_jsonable_python(trace.info[column])
+                elif hasattr(trace.task.data, column):
+                    record[column] = to_jsonable_python(
+                        getattr(trace.task.data, column)
+                    )
+            records.append(record)
+        return {comp: records for comp in components_to_update}

verifiers/v1/gepa/config.py

@@ -0,0 +1,78 @@
+"""The `GEPAConfig`: the single config object the `gepa` CLI parses.
+
+GEPA optimizes one taskset's `Task.system_prompt` by alternating rollouts (`evaluate`) with a
+teacher LM reflecting on the reflective dataset (`make_reflective_dataset`) — see
+`verifiers.v1.gepa.adapter.GEPAAdapter`. This inherits `EnvConfig`'s fields (`taskset`,
+`harness`, `max_turns`, token limits, timeouts) as top-level flags, the same way `EvalConfig`
+does, and adds the optimization loop's own knobs (model, reflection model, train/val split,
+budget). There is no worker pool here (`EnvServerConfig` is not a base) — GEPA always runs
+in-process, since its adapter protocol is itself synchronous (see `GEPAAdapter`)."""
+
+from pathlib import Path
+from uuid import uuid4
+
+from pydantic import AliasChoices, Field
+
+from verifiers.v1.clients import EvalClientConfig
+from verifiers.v1.env import EnvConfig
+from verifiers.v1.types import SamplingConfig
+
+
+class GEPAConfig(EnvConfig):
+    """The GEPA run plus its environment. `model` runs the rollouts under optimization;
+    `reflection_model` (defaults to `model`) proposes new system prompts from the reflective
+    dataset. No default for `model` — a GEPA run can spend a large eval budget, so pick it
+    explicitly rather than silently optimizing (and spending) against a fallback."""
+
+    uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
+    """Auto-generated run id — the leaf of the output dir, so runs never overwrite.
+    Excluded from the saved config so re-running `@ config.toml` lands in a fresh dir."""
+    model: str = Field(..., validation_alias=AliasChoices("model", "m"))
+    """Model id for rollouts under optimization."""
+    client: EvalClientConfig = EvalClientConfig()
+    sampling: SamplingConfig = SamplingConfig()
+    reflection_model: str | None = None
+    """Teacher model that proposes new system prompts. None = reuse `model`."""
+    reflection_client: EvalClientConfig | None = None
+    """Endpoint for `reflection_model`. None = reuse `client`."""
+
+    num_train: int = Field(100, ge=1)
+    """Tasks reserved for reflection minibatches (GEPA never scores the full trainset at once)."""
+    num_val: int = Field(50, ge=1)
+    """Tasks held out to score each candidate system prompt for the pareto frontier."""
+    shuffle: bool = Field(True, validation_alias=AliasChoices("shuffle", "s"))
+    """Shuffle tasks before splitting into train/val — v1 tasksets have no generic train/val
+    split, so GEPA carves one out of `taskset.load()` the way `run_eval` samples (fixed
+    seed, so the split is reproducible across runs)."""
+    seed: int = 0
+    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling
+    uses a fixed seed, matching eval — so this doesn't change the train/val split."""
+
+    max_metric_calls: int = Field(
+        500, validation_alias=AliasChoices("max_metric_calls", "B")
+    )
+    """Total rollouts GEPA may spend across the whole optimization run."""
+    reflection_minibatch_size: int = 3
+    """Train tasks sampled per reflection step."""
+    perfect_score: float | None = None
+    """Skip reflecting on a minibatch that already scores this well."""
+    reflection_columns: list[str] = Field(default_factory=list)
+    """Extra per-trace fields (from `trace.info`, else `task`) to surface to the teacher LM."""
+    initial_prompt: str | None = None
+    """Seed system prompt. None = the first loaded task's `Task.system_prompt`, if any task
+    sets one (see `resolve_gepa_seed_prompt`)."""
+
+    max_concurrent: int | None = Field(
+        128, validation_alias=AliasChoices("max_concurrent", "c")
+    )
+    """Max rollouts in flight at once, across the whole run."""
+    output_dir: Path | None = Field(
+        None, validation_alias=AliasChoices("output_dir", "o")
+    )
+    """Where to write results (config.toml + the streamed traces.jsonl, alongside GEPA's own
+    candidates.json / run_log.json). None = a fresh per-run dir under
+    `outputs/<taskset>--<model>--<harness>/<uuid>` (via `output_path`)."""
+    save_results: bool = True
+    verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
+    dry_run: bool = False
+    """Resolve + validate the config and dump it, then exit."""

verifiers/v1/gepa/dataset.py

@@ -0,0 +1,63 @@
+"""Train/val split and upfront validation for a GEPA run.
+
+v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
+individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of
+`taskset.load()` itself: the shared fixed-seed shuffle (`verifiers.v1.utils.sampling.sample`,
+reproducible across runs like every other entrypoint), then two disjoint slices.
+"""
+
+from verifiers.v1.decorators import discover_decorated
+from verifiers.v1.task import Task
+from verifiers.v1.utils.sampling import sample
+
+
+def reject_group_reward_tasksets(tasks: list[Task]) -> None:
+    """GEPA scores one rollout per task (`n=1`) to get the per-task scalar its pareto frontier
+    needs, but `@group_reward`s compare a task's rollouts and need >=2 — so a group-reward
+    taskset can't be optimized this way (`Environment.episode` would raise on the first batch).
+    Reject it up front with a clear message instead of crashing mid-run. A taskset loads one task
+    type, so the first task is representative (as `run_eval` also assumes)."""
+    if tasks and discover_decorated(tasks[0], "group_reward"):
+        raise ValueError(
+            "this taskset defines @group_reward(s), which compare >=2 rollouts of a task; GEPA "
+            "optimizes a single system prompt from per-task scalar scores (one rollout per task) "
+            "and can't score group rewards. Optimize a taskset without @group_reward instead."
+        )
+
+
+def split_tasks(
+    tasks: list[Task], num_train: int, num_val: int, shuffle: bool
+) -> tuple[list[Task], list[Task]]:
+    """The taskset's tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
+    minibatches; `val` scores each candidate for the pareto frontier)."""
+    pool = sample(
+        tasks, shuffle
+    )  # shared fixed-seed shuffle; GEPA does its own slicing below
+    if num_train + num_val > len(pool):
+        raise ValueError(
+            f"requested {num_train} train + {num_val} val tasks, but the taskset only "
+            f"loaded {len(pool)}"
+        )
+    return pool[:num_train], pool[num_train : num_train + num_val]
+
+
+def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> str:
+    """The system prompt GEPA starts optimizing from: `initial_prompt` if given, else the first
+    task that sets `system_prompt`. Some tasksets (e.g. `gsm8k-v1`) bake instructions into
+    `prompt` rather than `system_prompt` and can't be optimized this way — pass `--initial-prompt`
+    to seed one explicitly.
+
+    How the resolved prompt reaches the model at rollout time — a real system message vs. folded
+    into the user prompt — is the harness's call: `Harness.resolve_prompt` owns that policy and
+    warns when it folds, so it isn't re-policed here."""
+    if initial_prompt is not None:
+        return initial_prompt
+    for task in tasks:
+        if task.data.system_prompt is not None:
+            return task.data.system_prompt
+    raise ValueError(
+        "no task in this taskset sets Task.system_prompt — some tasksets bake instructions "
+        "directly into `prompt` instead (e.g. gsm8k-v1) and can't be optimized this way. Pass "
+        "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets "
+        "system_prompt on its task data (e.g. reverse-text-v1, lean, textarena)."
+    )

verifiers/v1/gepa/reflection.py

@@ -0,0 +1,28 @@
+"""The reflection (teacher) LM: GEPA's proposer calls this synchronously to turn a prompt into
+text. Backed by a vf `Judge` — `Judge.complete` owns the model client (same key / header /
+Prime-billing resolution rollouts use), so we never build one here. GEPA calls it from the
+worker thread running `optimize()`, so each call drives the async `complete` on a private loop.
+"""
+
+import asyncio
+from typing import Callable
+
+from verifiers.v1.gepa.config import GEPAConfig
+from verifiers.v1.judge import Judge, JudgeConfig
+
+
+def build_reflection_lm(config: GEPAConfig) -> Callable[[str], str]:
+    client = config.reflection_client or config.client
+    judge = Judge(
+        JudgeConfig(
+            model=config.reflection_model or config.model,
+            base_url=client.base_url,
+            api_key_var=client.api_key_var,
+            headers=client.headers,
+        )
+    )
+
+    def reflection_lm(prompt: str) -> str:
+        return asyncio.run(judge.complete(prompt)).text
+
+    return reflection_lm

verifiers/v1/gepa/runner.py

@@ -0,0 +1,122 @@
+"""The GEPA runner: split tasks, drive `gepa.api.optimize` against a `GEPAAdapter`, save
+results. Unlike the async sibling runners, `optimize()` is synchronous and blocking, so it runs
+on the main thread and the runner manages one event loop by hand: `env.serving()` is entered on
+it once, each rollout batch runs via `loop.run_until_complete` (see `GEPAAdapter.evaluate`),
+and everything is torn down in `finally`. Mirrors how v0 vf-gepa bridged sync GEPA to async
+rollouts — a Ctrl-C raises on the main thread inside `optimize()` and unwinds through teardown."""
+
+import asyncio
+import logging
+
+from gepa.api import optimize
+from gepa.core.result import GEPAResult
+
+from verifiers.v1.cli.output import append_trace, output_path, save_config
+from verifiers.v1.clients import ModelContext, resolve_client
+from verifiers.v1.env import Environment
+from verifiers.v1.gepa.adapter import GEPAAdapter
+from verifiers.v1.gepa.config import GEPAConfig
+from verifiers.v1.gepa.dataset import (
+    reject_group_reward_tasksets,
+    resolve_gepa_seed_prompt,
+    split_tasks,
+)
+from verifiers.v1.gepa.reflection import build_reflection_lm
+from verifiers.v1.trace import Trace
+
+logger = logging.getLogger(__name__)
+
+
+class _GEPALog:
+    """Forwards GEPA's optimizer log lines to v1 logging. GEPA's `LoggerProtocol` needs only
+    `.log(str)`, so its per-iteration progress rides the same loguru setup as the rest of the
+    v1 CLIs instead of a bespoke dashboard."""
+
+    def log(self, message: str) -> None:
+        logger.info(message)
+
+
+def run_gepa(env: Environment, config: GEPAConfig) -> GEPAResult:
+    logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
+    all_tasks = env.taskset.load()
+    train_tasks, val_tasks = split_tasks(
+        all_tasks, config.num_train, config.num_val, config.shuffle
+    )
+    selected_tasks = [*train_tasks, *val_tasks]
+    reject_group_reward_tasksets(
+        selected_tasks
+    )  # GEPA scores n=1 per task; groups need >=2
+    # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool —
+    # a taskset with per-task system prompts could otherwise seed from a task in neither split.
+    seed_prompt = resolve_gepa_seed_prompt(selected_tasks, config.initial_prompt)
+    tasks_by_idx = {task.data.idx: task for task in selected_tasks}
+
+    run_dir = output_path(config) if config.save_results else None
+    if run_dir is not None:
+        save_config(
+            config, run_dir
+        )  # config.toml + a fresh traces.jsonl (like run_eval)
+        logger.info("results: %s", run_dir)
+
+    # optimize() is synchronous and blocking, so it drives the run from this (main) thread. We
+    # own one event loop: `env.serving()` (shared tool servers + interception pool, built once
+    # like run_eval) is entered on it, each rollout batch runs on it via `loop.run_until_complete`
+    # (GEPAAdapter.evaluate), and it's all torn down in `finally`. Keeping optimize() on the
+    # main thread means a Ctrl-C raises straight through it into this teardown.
+    loop = asyncio.new_event_loop()
+    semaphore = (
+        asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
+    )
+    # Stream every rollout's trace to traces.jsonl as it finalizes — the same persist hook
+    # run_eval passes to Episode.run (each trace records its candidate prompt).
+    write_lock = asyncio.Lock()
+
+    async def on_complete(trace: Trace) -> None:
+        if run_dir is not None:
+            await append_trace(run_dir, trace, write_lock)
+
+    # The client opens an httpx pool at construction, so build it inside the try that closes it —
+    # a failure while building ctx/reflection_lm must not leak the pool.
+    client = None
+    try:
+        client = resolve_client(config.client)
+        ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
+        reflection_lm = build_reflection_lm(config)
+        serving = env.serving()
+        loop.run_until_complete(serving.__aenter__())
+        # Pair __aexit__ with a *successful* __aenter__: this inner block is reached only once
+        # serving is up, so a startup failure propagates as-is instead of being masked by
+        # tearing down a context that never entered.
+        try:
+            adapter = GEPAAdapter(
+                env=env,
+                ctx=ctx,
+                tasks=tasks_by_idx,
+                loop=loop,
+                semaphore=semaphore,
+                on_complete=on_complete,
+                reflection_columns=config.reflection_columns,
+            )
+            optimize_kwargs: dict = dict(
+                seed_candidate={"system_prompt": seed_prompt},
+                trainset=[task.data.idx for task in train_tasks],
+                valset=[task.data.idx for task in val_tasks],
+                adapter=adapter,
+                reflection_lm=reflection_lm,
+                max_metric_calls=config.max_metric_calls,
+                reflection_minibatch_size=config.reflection_minibatch_size,
+                run_dir=str(run_dir) if run_dir is not None else None,
+                seed=config.seed,
+                display_progress_bar=False,
+                skip_perfect_score=config.perfect_score is not None,
+                logger=_GEPALog(),
+            )
+            if config.perfect_score is not None:
+                optimize_kwargs["perfect_score"] = config.perfect_score
+            return optimize(**optimize_kwargs)
+        finally:
+            loop.run_until_complete(serving.__aexit__(None, None, None))
+    finally:
+        if client is not None:
+            loop.run_until_complete(client.close())
+        loop.close()

verifiers/v1/legacy.py

@@ -455,12 +455,12 @@ def _legacy_output_dir(config) -> Path:
 async def run_legacy_eval(config) -> list[Trace]:
     """Run a legacy environment in process and return v1 traces."""
     import asyncio
-    import random
 
     from verifiers import load_environment
 
     from verifiers.v1.cli.output import append_trace, save_config
     from verifiers.v1.utils.install import ensure_installed
+    from verifiers.v1.utils.sampling import sample
 
     # Install from the env hub on demand for an `org/name[@version]` id (a local id is
     # already importable), then load by module name.
@@ -468,11 +468,7 @@ async def run_legacy_eval(config) -> list[Trace]:
     if config.extra_env_kwargs:  # post-load knobs (max_total_completion_tokens, …)
         env.set_kwargs(**config.extra_env_kwargs)
     dataset = env.get_eval_dataset()  # the eval split (falls back to train when unset)
-    idxs = list(range(len(dataset)))
-    if config.shuffle:
-        random.Random(0).shuffle(idxs)  # fixed seed: same sample every run
-    if config.num_tasks is not None:
-        idxs = idxs[: config.num_tasks]
+    idxs = sample(list(range(len(dataset))), config.shuffle, config.num_tasks)
 
     client = _eval_client(config.client, config.model)
     sampling_args = config.sampling.model_dump(exclude_none=True)

verifiers/v1/utils/interrupt.py

@@ -1,6 +1,7 @@
-"""Graceful-shutdown signal handling for the eval CLI: the first Ctrl-C/SIGTERM warns and
-raises KeyboardInterrupt so asyncio unwinds each rollout's teardown `finally`; further signals
-are swallowed so a second Ctrl-C can't orphan containers/sandboxes mid-cleanup."""
+"""Graceful-shutdown signal handling shared across the v1 CLI entrypoints (eval, gepa,
+validate, debug, replay): the first Ctrl-C/SIGTERM warns and raises KeyboardInterrupt so
+asyncio unwinds each run's teardown `finally`; further signals are swallowed so a second
+Ctrl-C can't orphan containers/sandboxes/tool-server runtimes mid-cleanup."""
 
 import logging
 import signal
@@ -15,8 +16,9 @@ def cleaning_up() -> bool:
     return _cleaning_up
 
 
-def install() -> None:
-    """Route SIGINT/SIGTERM through the graceful-shutdown handler."""
+def install_interrupt() -> None:
+    """Route SIGINT/SIGTERM through the graceful-shutdown handler. Call once from a CLI
+    entrypoint before running its async main."""
 
     def handle(*_) -> None:
         global _cleaning_up

verifiers/v1/utils/sampling.py

@@ -0,0 +1,26 @@
+"""Shared task sampling: an optional fixed-seed shuffle, then an optional head-slice.
+
+Every taskset entrypoint — eval, server eval, debug, validate, legacy eval, and GEPA — narrows
+the loaded tasks (or their indices) the same way: with `--shuffle`, a shuffle under a fixed seed
+so the sampled subset is the *same* every run (reproducible), then an optional slice to the
+first `limit`. GEPA layers its disjoint train/val split on top of this shuffle (see
+`verifiers.v1.gepa.dataset`).
+"""
+
+import random
+from typing import TypeVar
+
+_SHUFFLE_SEED = (
+    0  # fixed so `--shuffle` samples the same items every run (reproducible)
+)
+
+T = TypeVar("T")
+
+
+def sample(items: list[T], shuffle: bool, limit: int | None = None) -> list[T]:
+    """`items` optionally shuffled under the fixed seed, then optionally sliced to the first
+    `limit`. Returns a new list; the input is left untouched."""
+    items = list(items)
+    if shuffle:
+        random.Random(_SHUFFLE_SEED).shuffle(items)
+    return items if limit is None else items[:limit]

{verifiers-0.2.1.dev6.dist-info → verifiers-0.2.1.dev7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev6
+Version: 0.2.1.dev7
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev6.dist-info → verifiers-0.2.1.dev7.dist-info}/RECORD

@@ -205,7 +205,7 @@ verifiers/v1/errors.py,sha256=GbFh_figoWzej2A1BEEB_quoaj1Xtt-rRIkb7KET1sA,6019
 verifiers/v1/graph.py,sha256=EVTjXUTrGIdcnekXdDpYYW7u5aXGLBPYUc0BeDjSmtA,25517
 verifiers/v1/harness.py,sha256=WochG6egKqhyf5qs_xWfUOPbVGWcE1mju9qZScxNmW0,5965
 verifiers/v1/judge.py,sha256=z7vLODlEwi5lk-TI92U46QQEyrl6clwQlg3R2uJiBFs,11094
-verifiers/v1/legacy.py,sha256=9WBVER4alu3Jy4lnhTZ9qht06JZCkB8O2g-3Kom9AsM,20560
+verifiers/v1/legacy.py,sha256=oQRMJOcc-vFSqJLUssUvyivLp1K-4FHEIwv4in7rI1Q,20459
 verifiers/v1/loaders.py,sha256=UBiwHl76cjgw41xsz07bc9hKg1cnT1c4m05G-R1ipt4,5044
 verifiers/v1/push.py,sha256=-jZdFjPTLuyBD5H_YeUYz5KFpbFHKzIhnW3BtiY_f8o,8545
 verifiers/v1/retries.py,sha256=oZYN0S5XQjvMHLxB_06iMNAzBMjlKotTwuQtgIGUgfw,5748
@@ -217,22 +217,23 @@ verifiers/v1/taskset.py,sha256=bOTITbhFqQS6XybjKdQE5nHuVlLOGObnwmJLRbFOh1A,2907
 verifiers/v1/trace.py,sha256=TZVV4J_nM-M5OeYvThW8RGrzaKdB3mTqk6QXw7vUL8U,15634
 verifiers/v1/types.py,sha256=rLeCLwzuN1sUzMSF2F_OqBOzS3Vd36aT7QbDf4xFfHQ,8264
 verifiers/v1/cli/__init__.py,sha256=x0ov9198kpwPlVd8IRbcbVtTYGAG5exo-IYocqa7gxw,23
-verifiers/v1/cli/debug.py,sha256=bpqXakBiKcJs8HlbupQjybLGHHj6gMoJRKEmHI3-B0M,10963
+verifiers/v1/cli/debug.py,sha256=duifS0H6CXfLsKJla1c-IwFr3zCErYZxfzy_eORLajQ,10962
+verifiers/v1/cli/gepa.py,sha256=jXzCVe4v10ogDOShN0lEXx5Dy1tu-ciiF_HXv4a3Byw,3183
 verifiers/v1/cli/init.py,sha256=foejcU68vsXHpVmpir3n-N3uKLX-CdsL-yI-9p9T4cg,9770
 verifiers/v1/cli/output.py,sha256=L0MaFqbGpFl_lOxNG7H43xKHF21zlwy4NoMb5aWBY3I,4026
-verifiers/v1/cli/replay.py,sha256=AN-Q4YRCSP_CE-b-2dDIFQNHY3osKDNXabqeY1K0lVc,9758
+verifiers/v1/cli/replay.py,sha256=H11cUNRKh9w41DygV8dqBm24fHpFFfyPxPKOnKsT_U0,9813
 verifiers/v1/cli/resolve.py,sha256=7xPR4enanS574I0GIcv30Hg0xX8OmC9GzVq93Mb9LS8,3441
 verifiers/v1/cli/serve.py,sha256=7T_pQdMihmGj7CdsEyVqlro_fBgdJVRkb3ak2qCkZgc,2240
-verifiers/v1/cli/validate.py,sha256=CjjNYXeS8z8Ow0eKcdiKCNVDiMCbOv-tH1mo2mutgpM,9513
+verifiers/v1/cli/validate.py,sha256=56CEagb7fvJQSj7qil-1wpH8ocCiPMalIJaF6PeTVUU,9427
 verifiers/v1/cli/dashboard/__init__.py,sha256=nWhXVYuAw-eF4TZbhtSoKWRQytv67WzIEnGISQnRHqY,227
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
 verifiers/v1/cli/dashboard/eval.py,sha256=hYRC_2HVlia87yR1MOGjMwzet1rVpeQzxZaKmo3wjRg,26166
 verifiers/v1/cli/dashboard/replay.py,sha256=4SmCFx7UbDbdRKszrPkOKSZQ303ezc9SqJLbAbClL6s,3119
 verifiers/v1/cli/dashboard/validate.py,sha256=1EyP2wkm0KMwl1Xf32my-ulmHwHHWzlHNgtbf-3ZQXs,3355
 verifiers/v1/cli/eval/__init__.py,sha256=64KcYxE9GDt_TB1CNuRI5uOx3A5npitYRKFuCHbV4DM,22
-verifiers/v1/cli/eval/main.py,sha256=48dGLSq8CZ48pjCiMRd8p1gRFcYXVwVinDVdYdThzfU,4931
+verifiers/v1/cli/eval/main.py,sha256=uch0w30MChcdofaprw48vGcJ2LXB-ms2Blb4bkXaQ6c,4920
 verifiers/v1/cli/eval/resume.py,sha256=CWKtKYmU6vtHD_rd4dfV4YV7ccgRNs7jBtA0zeOfHzA,5077
-verifiers/v1/cli/eval/runner.py,sha256=iSnocgsuZALpVCX0TewlHhisGFgnRoaIxePET70c7nM,10769
+verifiers/v1/cli/eval/runner.py,sha256=i7mzCOIF4mrKdWm6t5HqW7IB9-4cExKnt2h8bNX4208,10468
 verifiers/v1/clients/__init__.py,sha256=EgDgl5RYcjQ5W5ROjleL91eJoVfI36SsV9pSLrWVMVI,595
 verifiers/v1/clients/client.py,sha256=Y5Bvi13SNKC7jz4H8as9F98tzc7HY-VGxyo96_Y_FWQ,3247
 verifiers/v1/clients/config.py,sha256=QMR0PkpFLcLi_Zw6bVimigNjNiWeJ-4YvJ_JsSS9yGw,5228
@@ -250,6 +251,12 @@ verifiers/v1/dialects/anthropic.py,sha256=_WxY3EcZAx_HO-tMFpBz4ylcU9El3CPylRfLnb
 verifiers/v1/dialects/base.py,sha256=SIir2dCzlhhmjgqZv-fuw6aAgOQ8-OwOecEEvLc8UV4,8680
 verifiers/v1/dialects/chat.py,sha256=JJK0MH2_xygH8jYS6MCqPZ0-yKqLgOXMGDZCYD9F4P4,13326
 verifiers/v1/dialects/responses.py,sha256=d6FFXpAjrjxlaUwYVkywNC5iQZPCZp-vlTrABowC0ko,14592
+verifiers/v1/gepa/__init__.py,sha256=rZsikZtKKDKm7eU8hnn8fJi7FHScsTQJXNtFw1f-pD4,534
+verifiers/v1/gepa/adapter.py,sha256=-OyQwbsF6eNoql5TJ_qGs32zcTuUA1w4zNDvY-UfE5s,5336
+verifiers/v1/gepa/config.py,sha256=llwaOkWBBLKcCqDbeGXPJA1PWGtJZi3_47_FXXRolNk,4255
+verifiers/v1/gepa/dataset.py,sha256=YnoRxaDGKpyOwPQsvCOV1PlS9MjtaTBc6qFXlpaH14U,3355
+verifiers/v1/gepa/reflection.py,sha256=3wiMaXphu40i3nt38UhwNP06dMihqoVvaaBDO-52IR4,1007
+verifiers/v1/gepa/runner.py,sha256=xc9afhn2kHtqXb8p4-7EXROnNNYt8tm8Nfr7qpRZFcQ,5638
 verifiers/v1/harnesses/__init__.py,sha256=NF3TBQ9WYlTkODwYTEbqeM7zbwbRCW6onEfdQgL_Vqg,937
 verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
 verifiers/v1/harnesses/codex/harness.py,sha256=08T08uU-q9NSS2KWFQm5jaKaQ2Y-3NS3NfbKXqpRIVg,4525
@@ -307,11 +314,12 @@ verifiers/v1/utils/aio.py,sha256=ZKTHeURNbWhTpHMyYuqMLsdPWVHyn5anfG1IJs5y-Zg,148
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
 verifiers/v1/utils/generic.py,sha256=cpiyt_GIhFcHVK3dh7eBDqQJ6tQFXSQyHUuPKRLZfj4,647
 verifiers/v1/utils/install.py,sha256=fWNsyKrw_PyhC0Qhqv5Ri0adFm5Ddx4AVy_hbsra1GU,1390
-verifiers/v1/utils/interrupt.py,sha256=OTb6zQ9lpS3xQV5j4VDNiSB1N20fllaDbY8glI8Akx4,1027
+verifiers/v1/utils/interrupt.py,sha256=F-KKhc5ndPJJfhd3SuMqyqhXhA32FhCRy5KWFJpEoM4,1179
 verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y,2084
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
-verifiers-0.2.1.dev6.dist-info/METADATA,sha256=Ws2IpM5z_18GDHH6kmpnrN-HXxyAoME-RhpHA1ywnZs,5135
-verifiers-0.2.1.dev6.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-verifiers-0.2.1.dev6.dist-info/entry_points.txt,sha256=vYJT3h4I0-pEnXGaizK-6J56h-qrh-HvMtBTiATK06w,517
-verifiers-0.2.1.dev6.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.1.dev6.dist-info/RECORD,,
+verifiers/v1/utils/sampling.py,sha256=pzCf5HSYl41KFfX90hVNREwRkGncmcJYQ2ndRfVekI0,1037
+verifiers-0.2.1.dev7.dist-info/METADATA,sha256=HmlQhEtw1yRqv4_kNx1HrAtdkuBWYGPXETUU8g1Ydvk,5135
+verifiers-0.2.1.dev7.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+verifiers-0.2.1.dev7.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.1.dev7.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.1.dev7.dist-info/RECORD,,

{verifiers-0.2.1.dev6.dist-info → verifiers-0.2.1.dev7.dist-info}/entry_points.txt

@@ -1,6 +1,7 @@
 [console_scripts]
 debug = verifiers.v1.cli.debug:main
 eval = verifiers.v1.cli.eval.main:main
+gepa = verifiers.v1.cli.gepa:main
 init = verifiers.v1.cli.init:main
 replay = verifiers.v1.cli.replay:main
 serve = verifiers.v1.cli.serve:main