verifiers 0.2.1.dev5__tar.gz → 0.2.1.dev7__tar.gz

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev5
+Version: 0.2.1.dev7
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/pyproject.toml

@@ -170,6 +170,7 @@ debug = "verifiers.v1.cli.debug:main"
 replay = "verifiers.v1.cli.replay:main"
 serve = "verifiers.v1.cli.serve:main"
 init = "verifiers.v1.cli.init:main"
+gepa = "verifiers.v1.cli.gepa:main"
 vf-gepa = "verifiers.scripts.gepa:main"
 vf-init = "verifiers.scripts.init:main"
 vf-install = "verifiers.scripts.install:main"

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/verifiers/v1/cli/debug.py

@@ -3,9 +3,7 @@
 import asyncio
 import contextlib
 import logging
-import random
 import shlex
-import signal
 import sys
 import time
 import traceback
@@ -30,7 +28,9 @@ from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace, TraceTask
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -267,11 +267,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.load()
-    if config.shuffle:
-        random.Random(0).shuffle(tasks)
-    if config.num_tasks is not None:
-        tasks = tasks[: config.num_tasks]
+    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):
@@ -327,8 +323,9 @@ def main(argv: list[str] | None = None) -> None:
     sys.argv = [sys.argv[0], *argv]
     config = cli(config_type)
     setup_logging("DEBUG" if config.verbose else "INFO")
-    # Translate SIGTERM so async finally blocks still tear down their resources.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # Graceful shutdown: first Ctrl-C/SIGTERM unwinds each task's teardown `finally`;
+    # a second is swallowed so it can't orphan containers/sandboxes mid-cleanup.
+    install_interrupt()
     asyncio.run(run_debug(config))
 
 

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/verifiers/v1/cli/eval/main.py

@@ -7,7 +7,7 @@ import sys
 from pydantic_config import cli
 
 import verifiers.v1 as vf
-from verifiers.v1.utils.interrupt import install as install_interrupt
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 from verifiers.v1.cli.output import output_path, write_config
 from verifiers.v1.cli.resolve import (

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/verifiers/v1/cli/eval/runner.py

@@ -3,7 +3,6 @@
 import asyncio
 import contextlib
 import logging
-import random
 import time
 
 from verifiers.v1.clients import ModelContext, resolve_client
@@ -14,21 +13,15 @@ from verifiers.v1.cli.output import append_trace, output_path, save_config
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.env import Environment
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
-_SHUFFLE_SEED = (
-    0  # fixed so `--shuffle` samples the same tasks every run (reproducible)
-)
-
 
 async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     client = resolve_client(config.client)
-    tasks = env.taskset.load()
-    if config.shuffle:
-        random.Random(_SHUFFLE_SEED).shuffle(tasks)
-    tasks = tasks if config.num_tasks is None else tasks[: config.num_tasks]
+    tasks = sample(env.taskset.load(), config.shuffle, config.num_tasks)
     ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     # One episode of `num_rollouts` rollouts per task; the shared semaphore bounds total
     # concurrent rollouts (across episodes), so group rewards still see their whole episode.
@@ -169,11 +162,7 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
         await client.wait_for_server_startup(timeout=600)
         info = await client.info()
         group_scored = info.requires_group_scoring
-        idxs = list(range(info.num_tasks))
-        if config.shuffle:
-            random.Random(_SHUFFLE_SEED).shuffle(idxs)
-        if config.num_tasks is not None:
-            idxs = idxs[: config.num_tasks]
+        idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
         out = output_path(config)
         finished: list[Trace] = []
         if config.resume is not None:

verifiers-0.2.1.dev7/verifiers/v1/cli/gepa.py

@@ -0,0 +1,74 @@
+"""The GEPA entrypoint: `uv run gepa [<taskset-id>] --model <model> [options]`.
+
+Registered as the `gepa` console script. Optimizes a v1 taskset's `Task.system_prompt` via
+GEPA (Genetic-Pareto): alternating rollouts with a teacher LM reflecting on results — see
+`verifiers.v1.gepa`. CLI resolution mirrors `eval`/`serve` (`verifiers.v1.cli.resolve`): a
+leading bare token is the taskset id, the taskset/harness subconfigs are narrowed from their
+`id`s so `--taskset.*` / `--harness.*` stay typed and `-h` renders them, `@ file.toml` loads,
+and the actual parse is `pydantic_config.cli`. v1-native tasksets only — a legacy (v0) env is
+rejected; run those through the existing `vf-gepa` command instead.
+"""
+
+import logging
+import sys
+
+from pydantic_config import cli
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.output import output_path, write_config
+from verifiers.v1.cli.resolve import (
+    extract_id,
+    narrow_config,
+    references_config_file,
+    with_positional_taskset,
+)
+from verifiers.v1.gepa import GEPAConfig, run_gepa
+from verifiers.v1.utils.interrupt import install_interrupt
+from verifiers.v1.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+USAGE = "usage: uv run gepa [<taskset-id>] [--harness.id <id>] --model <model> [options] [@ file.toml]"
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
+
+    if not argv or any(arg in ("-h", "--help") for arg in argv):
+        print(USAGE)
+        sys.argv = [sys.argv[0], "--help"]
+        cli(
+            narrow_config(GEPAConfig, argv)
+        )  # full option help, narrowed to the given ids
+        return
+    if not extract_id(argv, "taskset") and not references_config_file(argv):
+        raise SystemExit(
+            USAGE
+        )  # need a taskset (positional / --taskset.id) or a @ file.toml
+
+    config_type = narrow_config(GEPAConfig, argv)
+    sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
+    config = cli(config_type)
+    if config.is_legacy:
+        raise SystemExit(
+            "gepa optimizes native v1 tasksets; run a legacy (v0) environment through "
+            "`vf-gepa` instead of `gepa`."
+        )
+    setup_logging("DEBUG" if config.verbose else "INFO")
+    if config.dry_run:  # resolved + validated; write it to the output dir and exit
+        logger.info("wrote config to %s", write_config(config, output_path(config)))
+        return
+
+    # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out run still
+    # runs the runner's `serving()` teardown (interception pool / tool-server runtimes); further
+    # signals during that cleanup are swallowed so an impatient second Ctrl-C can't orphan them.
+    install_interrupt()
+
+    env = vf.Environment(config)
+    try:
+        result = run_gepa(env, config)
+    except KeyboardInterrupt:
+        # Graceful cleanup already ran (run_gepa's finally tore down serving); exit on the
+        # conventional Ctrl-C code without dumping a traceback.
+        raise SystemExit(130)
+    print(f"best system prompt:\n{result.best_candidate.get('system_prompt', '')}")

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/verifiers/v1/cli/replay.py

@@ -10,7 +10,6 @@ config is the base for replay-specific overrides.
 import asyncio
 import contextlib
 import logging
-import signal
 import sys
 import time
 import tomllib
@@ -31,6 +30,7 @@ from verifiers.v1.configs.replay import ReplayConfig
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task, WireTaskData, task_data_cls
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
@@ -220,8 +220,9 @@ def main(argv: list[str] | None = None) -> None:
         logging.lastResort = None
     else:
         setup_logging(level, log_file=log_file, console=True)
-    # Translate SIGTERM so async finally blocks still tear down their resources.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # Graceful shutdown: first Ctrl-C/SIGTERM unwinds the scoring teardown `finally`;
+    # a second is swallowed so it can't orphan resources mid-cleanup.
+    install_interrupt()
     asyncio.run(run_replay(config, source, out))
 
 

{verifiers-0.2.1.dev5 → verifiers-0.2.1.dev7}/verifiers/v1/cli/validate.py

@@ -3,8 +3,6 @@
 import asyncio
 import contextlib
 import logging
-import random
-import signal
 import sys
 import time
 from typing import Any
@@ -26,7 +24,9 @@ from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
+from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -198,11 +198,7 @@ async def _validate_task(task: Task, config: ValidateConfig) -> ResultRow:
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
     taskset = vf.load_taskset(config.taskset)
-    tasks = taskset.load()
-    if config.shuffle:
-        random.Random(0).shuffle(tasks)
-    if config.num_tasks is not None:
-        tasks = tasks[: config.num_tasks]
+    tasks = sample(taskset.load(), config.shuffle, config.num_tasks)
     if isinstance(config.runtime, vf.SubprocessConfig) and any(
         type(t).NEEDS_CONTAINER or t.data.image for t in tasks
     ):
@@ -273,9 +269,9 @@ def main(argv: list[str] | None = None) -> None:
     setup_logging("DEBUG" if config.verbose else "INFO", console=not config.rich)
     if config.rich:
         logging.lastResort = None  # drop stdlib records that bypass loguru
-    # Make SIGTERM behave like Ctrl-C so a killed run still runs each task's `finally`
-    # (tears down containers/sandboxes) — and the atexit backstop catches the rest.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # Graceful shutdown: first Ctrl-C/SIGTERM unwinds each task's teardown `finally`
+    # (containers/sandboxes); a second is swallowed so it can't orphan them mid-cleanup.
+    install_interrupt()
     asyncio.run(run_validate(config))
 
 

verifiers-0.2.1.dev7/verifiers/v1/gepa/__init__.py

@@ -0,0 +1,12 @@
+"""GEPA (Genetic-Pareto) system-prompt optimization for native v1 environments.
+
+`run_gepa(env, config)` drives the third-party `gepa` optimizer against a v1 `Environment`
+via `GEPAAdapter`, seeding from and improving `Task.system_prompt`. The `gepa` console
+script (`verifiers.v1.cli.gepa`) is a thin entrypoint over this package.
+"""
+
+from verifiers.v1.gepa.adapter import GEPAAdapter
+from verifiers.v1.gepa.config import GEPAConfig
+from verifiers.v1.gepa.runner import run_gepa
+
+__all__ = ["GEPAAdapter", "GEPAConfig", "run_gepa"]

verifiers-0.2.1.dev7/verifiers/v1/gepa/adapter.py

@@ -0,0 +1,112 @@
+"""The GEPA <-> v1 bridge: run a candidate system prompt over a batch of tasks and score it.
+
+GEPA's adapter protocol (`evaluate`, `make_reflective_dataset`) is synchronous and
+`gepa.api.optimize` blocks, but v1 rollouts are async. The runner manages one event loop by
+hand: it enters `env.serving()` on that loop and runs the blocking `optimize()` on the main
+thread, so each synchronous `evaluate()` drives its batch of rollouts with
+`loop.run_until_complete` — the one sync↔async hop. (Mirrors how v0 vf-gepa bridged; no worker
+thread, so a Ctrl-C unwinds straight through `optimize()` into the runner's teardown.)
+"""
+
+import asyncio
+from collections.abc import Awaitable
+from dataclasses import dataclass, field
+from typing import Any, Callable, Mapping, Sequence
+
+from gepa.core.adapter import EvaluationBatch
+from pydantic_core import to_jsonable_python
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.env import Environment
+from verifiers.v1.task import Task
+from verifiers.v1.trace import Trace
+
+Candidate = dict[str, str]
+
+
+@dataclass
+class GEPAAdapter:
+    """Bridges GEPA's optimization loop with a native v1 `Environment`. `tasks` covers only the
+    trainset + valset tasks GEPA was given (not the whole taskset), keyed by `task.data.idx` —
+    GEPA's `batch` is a list of those idxs, and injecting the candidate rebuilds each `Task`
+    around a `data` row carrying the new `system_prompt`. `loop` is the runner's persistent event
+    loop (which holds `env.serving()` open); `evaluate` drives its rollouts on it with
+    `run_until_complete`."""
+
+    env: Environment
+    ctx: ModelContext
+    tasks: dict[int, Task]
+    loop: asyncio.AbstractEventLoop
+    semaphore: asyncio.Semaphore | None = None
+    on_complete: Callable[[Trace], Awaitable[None]] | None = None
+    """Called with each rollout's trace as it finalizes — the runner's persist hook that
+    streams traces to `traces.jsonl`, exactly as `run_eval` does."""
+    reflection_columns: list[str] = field(default_factory=list)
+    propose_new_texts: Callable[..., Candidate] | None = None
+    """Part of GEPA's adapter protocol — its proposer reads this attribute on every reflection
+    step. None = use GEPA's default reflection-LM proposer (the AttributeError from leaving it
+    undeclared silently disables all mutation proposals)."""
+
+    def evaluate(
+        self,
+        batch: list[int],
+        candidate: Candidate,
+        capture_traces: bool = False,
+    ) -> EvaluationBatch[Trace, Trace]:
+        """Run `candidate`'s system prompt on the tasks named by `batch` (`Task.idx` values)
+        and score them. Called synchronously by GEPA on the main thread; each batch's rollouts
+        run on the runner's persistent loop via `run_until_complete`."""
+        system_prompt = candidate.get("system_prompt", "")
+        traces = self.loop.run_until_complete(self._run_batch(batch, system_prompt))
+        scores = [trace.reward for trace in traces]
+        return EvaluationBatch(
+            outputs=traces,
+            scores=scores,
+            trajectories=traces if capture_traces else None,
+        )
+
+    async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Trace]:
+        # Inject the candidate by rebuilding each Task around a data row with the new
+        # system_prompt (TaskData is frozen; behavior/config carry over unchanged).
+        tasks = [
+            type(t)(
+                t.data.model_copy(update={"system_prompt": system_prompt}), t.config
+            )
+            for t in (self.tasks[idx] for idx in batch)
+        ]
+        episodes = [self.env.episode(task, self.ctx, n=1) for task in tasks]
+        results = await asyncio.gather(
+            *(episode.run(self.semaphore, self.on_complete) for episode in episodes)
+        )
+        return [trace for episode_traces in results for trace in episode_traces]
+
+    def make_reflective_dataset(
+        self,
+        candidate: Candidate,  # noqa: ARG002 - required by GEPA's adapter protocol
+        eval_batch: EvaluationBatch[Trace, Trace],
+        components_to_update: list[str],
+    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
+        """Build the reflective dataset the teacher LM reads to propose a new system prompt,
+        from `eval_batch.trajectories` (traces captured by a prior `evaluate(capture_traces=True)`
+        on the same batch)."""
+        traces = eval_batch.trajectories or []
+        records = []
+        for trace, score in zip(traces, eval_batch.scores):
+            record: dict[str, Any] = {
+                "query": trace.task.data.prompt_text,
+                "completion": trace.last_reply,
+                "reward": score,
+            }
+            if trace.has_error:
+                record["error"] = str(trace.error)
+            if trace.stop_condition:
+                record["stop_condition"] = trace.stop_condition
+            for column in self.reflection_columns:
+                if column in trace.info:
+                    record[column] = to_jsonable_python(trace.info[column])
+                elif hasattr(trace.task.data, column):
+                    record[column] = to_jsonable_python(
+                        getattr(trace.task.data, column)
+                    )
+            records.append(record)
+        return {comp: records for comp in components_to_update}

verifiers-0.2.1.dev7/verifiers/v1/gepa/config.py

@@ -0,0 +1,78 @@
+"""The `GEPAConfig`: the single config object the `gepa` CLI parses.
+
+GEPA optimizes one taskset's `Task.system_prompt` by alternating rollouts (`evaluate`) with a
+teacher LM reflecting on the reflective dataset (`make_reflective_dataset`) — see
+`verifiers.v1.gepa.adapter.GEPAAdapter`. This inherits `EnvConfig`'s fields (`taskset`,
+`harness`, `max_turns`, token limits, timeouts) as top-level flags, the same way `EvalConfig`
+does, and adds the optimization loop's own knobs (model, reflection model, train/val split,
+budget). There is no worker pool here (`EnvServerConfig` is not a base) — GEPA always runs
+in-process, since its adapter protocol is itself synchronous (see `GEPAAdapter`)."""
+
+from pathlib import Path
+from uuid import uuid4
+
+from pydantic import AliasChoices, Field
+
+from verifiers.v1.clients import EvalClientConfig
+from verifiers.v1.env import EnvConfig
+from verifiers.v1.types import SamplingConfig
+
+
+class GEPAConfig(EnvConfig):
+    """The GEPA run plus its environment. `model` runs the rollouts under optimization;
+    `reflection_model` (defaults to `model`) proposes new system prompts from the reflective
+    dataset. No default for `model` — a GEPA run can spend a large eval budget, so pick it
+    explicitly rather than silently optimizing (and spending) against a fallback."""
+
+    uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
+    """Auto-generated run id — the leaf of the output dir, so runs never overwrite.
+    Excluded from the saved config so re-running `@ config.toml` lands in a fresh dir."""
+    model: str = Field(..., validation_alias=AliasChoices("model", "m"))
+    """Model id for rollouts under optimization."""
+    client: EvalClientConfig = EvalClientConfig()
+    sampling: SamplingConfig = SamplingConfig()
+    reflection_model: str | None = None
+    """Teacher model that proposes new system prompts. None = reuse `model`."""
+    reflection_client: EvalClientConfig | None = None
+    """Endpoint for `reflection_model`. None = reuse `client`."""
+
+    num_train: int = Field(100, ge=1)
+    """Tasks reserved for reflection minibatches (GEPA never scores the full trainset at once)."""
+    num_val: int = Field(50, ge=1)
+    """Tasks held out to score each candidate system prompt for the pareto frontier."""
+    shuffle: bool = Field(True, validation_alias=AliasChoices("shuffle", "s"))
+    """Shuffle tasks before splitting into train/val — v1 tasksets have no generic train/val
+    split, so GEPA carves one out of `taskset.load()` the way `run_eval` samples (fixed
+    seed, so the split is reproducible across runs)."""
+    seed: int = 0
+    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling
+    uses a fixed seed, matching eval — so this doesn't change the train/val split."""
+
+    max_metric_calls: int = Field(
+        500, validation_alias=AliasChoices("max_metric_calls", "B")
+    )
+    """Total rollouts GEPA may spend across the whole optimization run."""
+    reflection_minibatch_size: int = 3
+    """Train tasks sampled per reflection step."""
+    perfect_score: float | None = None
+    """Skip reflecting on a minibatch that already scores this well."""
+    reflection_columns: list[str] = Field(default_factory=list)
+    """Extra per-trace fields (from `trace.info`, else `task`) to surface to the teacher LM."""
+    initial_prompt: str | None = None
+    """Seed system prompt. None = the first loaded task's `Task.system_prompt`, if any task
+    sets one (see `resolve_gepa_seed_prompt`)."""
+
+    max_concurrent: int | None = Field(
+        128, validation_alias=AliasChoices("max_concurrent", "c")
+    )
+    """Max rollouts in flight at once, across the whole run."""
+    output_dir: Path | None = Field(
+        None, validation_alias=AliasChoices("output_dir", "o")
+    )
+    """Where to write results (config.toml + the streamed traces.jsonl, alongside GEPA's own
+    candidates.json / run_log.json). None = a fresh per-run dir under
+    `outputs/<taskset>--<model>--<harness>/<uuid>` (via `output_path`)."""
+    save_results: bool = True
+    verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
+    dry_run: bool = False
+    """Resolve + validate the config and dump it, then exit."""

verifiers-0.2.1.dev7/verifiers/v1/gepa/dataset.py

@@ -0,0 +1,63 @@
+"""Train/val split and upfront validation for a GEPA run.
+
+v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
+individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of
+`taskset.load()` itself: the shared fixed-seed shuffle (`verifiers.v1.utils.sampling.sample`,
+reproducible across runs like every other entrypoint), then two disjoint slices.
+"""
+
+from verifiers.v1.decorators import discover_decorated
+from verifiers.v1.task import Task
+from verifiers.v1.utils.sampling import sample
+
+
+def reject_group_reward_tasksets(tasks: list[Task]) -> None:
+    """GEPA scores one rollout per task (`n=1`) to get the per-task scalar its pareto frontier
+    needs, but `@group_reward`s compare a task's rollouts and need >=2 — so a group-reward
+    taskset can't be optimized this way (`Environment.episode` would raise on the first batch).
+    Reject it up front with a clear message instead of crashing mid-run. A taskset loads one task
+    type, so the first task is representative (as `run_eval` also assumes)."""
+    if tasks and discover_decorated(tasks[0], "group_reward"):
+        raise ValueError(
+            "this taskset defines @group_reward(s), which compare >=2 rollouts of a task; GEPA "
+            "optimizes a single system prompt from per-task scalar scores (one rollout per task) "
+            "and can't score group rewards. Optimize a taskset without @group_reward instead."
+        )
+
+
+def split_tasks(
+    tasks: list[Task], num_train: int, num_val: int, shuffle: bool
+) -> tuple[list[Task], list[Task]]:
+    """The taskset's tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
+    minibatches; `val` scores each candidate for the pareto frontier)."""
+    pool = sample(
+        tasks, shuffle
+    )  # shared fixed-seed shuffle; GEPA does its own slicing below
+    if num_train + num_val > len(pool):
+        raise ValueError(
+            f"requested {num_train} train + {num_val} val tasks, but the taskset only "
+            f"loaded {len(pool)}"
+        )
+    return pool[:num_train], pool[num_train : num_train + num_val]
+
+
+def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> str:
+    """The system prompt GEPA starts optimizing from: `initial_prompt` if given, else the first
+    task that sets `system_prompt`. Some tasksets (e.g. `gsm8k-v1`) bake instructions into
+    `prompt` rather than `system_prompt` and can't be optimized this way — pass `--initial-prompt`
+    to seed one explicitly.
+
+    How the resolved prompt reaches the model at rollout time — a real system message vs. folded
+    into the user prompt — is the harness's call: `Harness.resolve_prompt` owns that policy and
+    warns when it folds, so it isn't re-policed here."""
+    if initial_prompt is not None:
+        return initial_prompt
+    for task in tasks:
+        if task.data.system_prompt is not None:
+            return task.data.system_prompt
+    raise ValueError(
+        "no task in this taskset sets Task.system_prompt — some tasksets bake instructions "
+        "directly into `prompt` instead (e.g. gsm8k-v1) and can't be optimized this way. Pass "
+        "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets "
+        "system_prompt on its task data (e.g. reverse-text-v1, lean, textarena)."
+    )

verifiers-0.2.1.dev7/verifiers/v1/gepa/reflection.py

@@ -0,0 +1,28 @@
+"""The reflection (teacher) LM: GEPA's proposer calls this synchronously to turn a prompt into
+text. Backed by a vf `Judge` — `Judge.complete` owns the model client (same key / header /
+Prime-billing resolution rollouts use), so we never build one here. GEPA calls it from the
+worker thread running `optimize()`, so each call drives the async `complete` on a private loop.
+"""
+
+import asyncio
+from typing import Callable
+
+from verifiers.v1.gepa.config import GEPAConfig
+from verifiers.v1.judge import Judge, JudgeConfig
+
+
+def build_reflection_lm(config: GEPAConfig) -> Callable[[str], str]:
+    client = config.reflection_client or config.client
+    judge = Judge(
+        JudgeConfig(
+            model=config.reflection_model or config.model,
+            base_url=client.base_url,
+            api_key_var=client.api_key_var,
+            headers=client.headers,
+        )
+    )
+
+    def reflection_lm(prompt: str) -> str:
+        return asyncio.run(judge.complete(prompt)).text
+
+    return reflection_lm