verifiers 0.4.2.dev34__py3-none-any.whl → 0.4.2.dev44__py3-none-any.whl

verifiers/v1/clients/eval.py

@@ -9,6 +9,7 @@ from pydantic_core import from_json, to_json
 
 from verifiers.v1.clients.base import build_async_httpx, join_url
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client, RelayReply
+from verifiers.v1.clients.prompt_cache import maybe_add_prompt_cache_control
 from verifiers.v1.configs.client import (
     BaseClientConfig,
     resolve_api_key,
@@ -70,6 +71,7 @@ class EvalClient(Client):
     def __init__(self, config: BaseClientConfig) -> None:
         self.base_url = config.base_url
         self.api_key = resolve_api_key(config)
+        self.cache_retention = getattr(config, "cache_retention", None)
         # Keep endpoint headers separate so they can override intercepted request headers before
         # the dialect's provider authentication is applied.
         self.headers = resolve_headers(config)
@@ -84,6 +86,9 @@ class EvalClient(Client):
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> Response:
+        body = maybe_add_prompt_cache_control(
+            dialect, body, self.base_url, self.cache_retention
+        )
         resp = await self._request(
             join_url(self.base_url, dialect.upstream_path),
             body,
@@ -177,6 +182,9 @@ class EvalClient(Client):
     ) -> RelayReply:
         # Relay complete SSE events so the interception server can safely insert keepalives
         # between them. Error responses are mapped before any event is handed back.
+        body = maybe_add_prompt_cache_control(
+            dialect, body, self.base_url, self.cache_retention
+        )
         resp = await self._request(
             join_url(self.base_url, dialect.upstream_path),
             body,

verifiers/v1/clients/prompt_cache.py

@@ -0,0 +1,350 @@
+"""Gateway-side prompt-cache markers for Anthropic models.
+
+Anthropic has no implicit prompt cache: a request only caches content when a
+block carries ``cache_control``. Verifiers relays harness-native bodies, and
+many harnesses send no marker, so Claude traffic through Prime Inference or
+OpenRouter pays full input price. When the caller sends no marker, mark the
+static prefix (last custom tool and last system/developer block) that repeats
+across rollouts; multi-turn requests also mark the latest message.
+
+Markers default to the five-minute cache (no ``ttl``). Set
+``cache_retention = "long"`` on the eval client, or ``PI_CACHE_RETENTION=long``,
+to emit ``ttl: "1h"`` instead; one-hour writes cost twice as much. ``"none"``
+disables injection entirely. Any caller marker wins over generation here.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from typing import Any, Literal
+from urllib.parse import urlsplit
+
+from verifiers.v1.dialects.base import Dialect
+
+CacheRetention = Literal["none", "short", "long"]
+
+_CACHE_MARKERS = frozenset({"cache_control", "prompt_cache_breakpoint"})
+_SCHEMA_KEYS = frozenset({"input_schema", "parameters", "json_schema"})
+_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_result", "document"})
+
+
+def _has_marker(value: Any) -> bool:
+    return isinstance(value, Mapping) and not _CACHE_MARKERS.isdisjoint(value)
+
+
+def _content_has_marker(content: Any) -> bool:
+    if isinstance(content, Mapping):
+        return _has_marker(content)
+    if not isinstance(content, list):
+        return False
+    return any(isinstance(block, Mapping) and _has_marker(block) for block in content)
+
+
+def _has_caller_marker(body: Mapping[str, Any]) -> bool:
+    """Only scan positions where Anthropic honors cache markers.
+
+    Tool-use inputs, tool arguments, metadata, and JSON schemas can contain a
+    ``cache_control`` key without being prompt-cache markers.
+    """
+    if _has_marker(body):
+        return True
+    if _content_has_marker(body.get("system")):
+        return True
+    messages = body.get("messages")
+    if isinstance(messages, list):
+        for message in messages:
+            if not isinstance(message, Mapping):
+                continue
+            if _has_marker(message):
+                return True
+            content = message.get("content")
+            if _content_has_marker(content):
+                return True
+            if isinstance(content, list):
+                for block in content:
+                    if (
+                        isinstance(block, Mapping)
+                        and block.get("type") == "tool_result"
+                        and _content_has_marker(block.get("content"))
+                    ):
+                        return True
+    tools = body.get("tools")
+    if isinstance(tools, list):
+        for tool in tools:
+            if _has_marker(tool):
+                return True
+    return False
+
+
+def resolve_cache_retention(
+    cache_retention: CacheRetention | None,
+) -> CacheRetention:
+    """Explicit setting wins, then ``PI_CACHE_RETENTION=long``, then short."""
+    if cache_retention is not None:
+        return cache_retention
+    env = os.environ.get("PI_CACHE_RETENTION", "").strip().casefold()
+    if env in {"none", "short", "long"}:
+        return env
+    return "short"
+
+
+def _cache_marker(retention: CacheRetention) -> dict[str, str]:
+    marker = {"type": "ephemeral"}
+    if retention == "long":
+        marker["ttl"] = "1h"
+    return marker
+
+
+def _is_anthropic_model(body: Mapping[str, Any]) -> bool:
+    model = body.get("model")
+    return isinstance(model, str) and model.casefold().startswith("anthropic/")
+
+
+def _is_gateway(base_url: str) -> bool:
+    hostname = urlsplit(base_url).hostname
+    if hostname is None:
+        return False
+    host = hostname.casefold()
+    return host in {"openrouter.ai", "pinference.ai"} or host.endswith(
+        (".openrouter.ai", ".pinference.ai")
+    )
+
+
+def _mark_last_tool(
+    body: dict[str, Any], *, anthropic: bool, marker: dict[str, str]
+) -> bool:
+    tools = body.get("tools")
+    if not isinstance(tools, list) or not tools:
+        return False
+    for index in range(len(tools) - 1, -1, -1):
+        tool = tools[index]
+        if not isinstance(tool, dict):
+            continue
+        if anthropic:
+            # Server tools (web search, computer use, ...) do not accept a
+            # cache marker; fall back to the last custom tool.
+            tool_type = tool.get("type")
+            if (tool_type is not None and tool_type != "custom") or "name" not in tool:
+                continue
+        elif tool.get("type") != "function":
+            continue
+        updated = list(tools)
+        updated[index] = {**tool, "cache_control": dict(marker)}
+        body["tools"] = updated
+        return True
+    return False
+
+
+def _mark_last_text_block(
+    message: dict[str, Any], marker: dict[str, str]
+) -> dict | None:
+    content = message.get("content")
+    if isinstance(content, str):
+        if not content:
+            return None
+        return {
+            **message,
+            "content": [
+                {
+                    "type": "text",
+                    "text": content,
+                    "cache_control": dict(marker),
+                }
+            ],
+        }
+    if not isinstance(content, list):
+        return None
+    for index in range(len(content) - 1, -1, -1):
+        block = content[index]
+        if (
+            isinstance(block, dict)
+            and block.get("type") == "text"
+            and isinstance(block.get("text"), str)
+            and block["text"]
+        ):
+            updated = list(content)
+            updated[index] = {**block, "cache_control": dict(marker)}
+            return {**message, "content": updated}
+    return None
+
+
+def _mark_last_anthropic_block(
+    message: dict[str, Any], marker: dict[str, str]
+) -> dict | None:
+    content = message.get("content")
+    if isinstance(content, str):
+        if not content:
+            return None
+        return {
+            **message,
+            "content": [
+                {
+                    "type": "text",
+                    "text": content,
+                    "cache_control": dict(marker),
+                }
+            ],
+        }
+    if not isinstance(content, list):
+        return None
+    for index in range(len(content) - 1, -1, -1):
+        block = content[index]
+        if not isinstance(block, dict):
+            continue
+        block_type = block.get("type")
+        if not isinstance(block_type, str) or block_type not in _ANTHROPIC_BLOCK_TYPES:
+            continue
+        if block_type == "text" and not (
+            isinstance(block.get("text"), str) and block["text"]
+        ):
+            continue
+        updated = list(content)
+        updated[index] = {**block, "cache_control": dict(marker)}
+        return {**message, "content": updated}
+    return None
+
+
+def _leading_instruction_indices(messages: list[Any]) -> list[int]:
+    indices: list[int] = []
+    for index, message in enumerate(messages):
+        if not isinstance(message, dict):
+            break
+        role = message.get("role")
+        if not isinstance(role, str) or role not in {"system", "developer"}:
+            break
+        indices.append(index)
+    return indices
+
+
+def _has_chat_history(messages: list[Any]) -> bool:
+    user_turns = 0
+    for message in messages:
+        if not isinstance(message, dict):
+            continue
+        role = message.get("role")
+        if not isinstance(role, str):
+            continue
+        if role == "user":
+            user_turns += 1
+        elif role in {"assistant", "tool"}:
+            return True
+    return user_turns > 1
+
+
+def _has_messages_history(messages: list[Any]) -> bool:
+    if len([message for message in messages if isinstance(message, dict)]) <= 1:
+        return False
+    return isinstance(messages[-1], dict) and messages[-1].get("role") == "user"
+
+
+def _mark_chat(body: dict[str, Any], marker: dict[str, str]) -> bool:
+    applied = _mark_last_tool(body, anthropic=False, marker=marker)
+    messages = body.get("messages")
+    if not isinstance(messages, list) or not messages:
+        return applied
+
+    updated_messages = list(messages)
+    messages_changed = False
+    for index in reversed(_leading_instruction_indices(messages)):
+        marked = _mark_last_text_block(messages[index], marker)
+        if marked is not None:
+            updated_messages[index] = marked
+            messages_changed = True
+            applied = True
+            break
+
+    last_message = messages[-1] if messages else None
+    last_role = last_message.get("role") if isinstance(last_message, dict) else None
+    if (
+        _has_chat_history(messages)
+        and isinstance(last_message, dict)
+        and isinstance(last_role, str)
+        and last_role in {"user", "tool"}
+    ):
+        marked = _mark_last_text_block(last_message, marker)
+        if marked is not None:
+            updated_messages[-1] = marked
+            messages_changed = True
+            applied = True
+
+    if messages_changed:
+        body["messages"] = updated_messages
+    return applied
+
+
+def _mark_anthropic_messages(body: dict[str, Any], marker: dict[str, str]) -> bool:
+    applied = _mark_last_tool(body, anthropic=True, marker=marker)
+
+    system = body.get("system")
+    if isinstance(system, str):
+        if system:
+            body["system"] = [
+                {
+                    "type": "text",
+                    "text": system,
+                    "cache_control": dict(marker),
+                }
+            ]
+            applied = True
+    elif isinstance(system, list):
+        for index in range(len(system) - 1, -1, -1):
+            block = system[index]
+            if (
+                isinstance(block, dict)
+                and block.get("type") == "text"
+                and isinstance(block.get("text"), str)
+                and block["text"]
+            ):
+                updated = list(system)
+                updated[index] = {**block, "cache_control": dict(marker)}
+                body["system"] = updated
+                applied = True
+                break
+
+    messages = body.get("messages")
+    if isinstance(messages, list) and messages and _has_messages_history(messages):
+        marked = _mark_last_anthropic_block(messages[-1], marker)
+        if marked is not None:
+            body["messages"] = [*messages[:-1], marked]
+            applied = True
+    return applied
+
+
+def maybe_add_prompt_cache_control(
+    dialect: Dialect,
+    body: dict[str, Any],
+    base_url: str,
+    cache_retention: CacheRetention | None = None,
+) -> dict[str, Any]:
+    """Return a request body with prompt-cache markers when eligible.
+
+    Only Anthropic models on Prime Inference/OpenRouter are touched, and only
+    when the caller has not already set any cache marker. The original body is
+    left untouched; edited lists and blocks are copied.
+    """
+    if dialect.name not in {"anthropic_messages", "chat_completions"}:
+        return body
+    if not _is_gateway(base_url) or not _is_anthropic_model(body):
+        return body
+    retention = resolve_cache_retention(cache_retention)
+    if retention == "none":
+        return body
+    if _has_caller_marker(body):
+        return body
+
+    marker = _cache_marker(retention)
+    updated = dict(body)
+    if isinstance(body.get("tools"), list):
+        updated["tools"] = list(body["tools"])
+    if isinstance(body.get("messages"), list):
+        updated["messages"] = list(body["messages"])
+    if isinstance(body.get("system"), list):
+        updated["system"] = list(body["system"])
+
+    applied = (
+        _mark_anthropic_messages(updated, marker)
+        if dialect.name == "anthropic_messages"
+        else _mark_chat(updated, marker)
+    )
+    return updated if applied else body

verifiers/v1/configs/cli/env.py

@@ -3,8 +3,8 @@ every config that owns one. `merge_defaults` (`utils/generic.py`) layers shared
 defaults under it.
 
 A run composes the blocks it needs — `[env]` (what runs, `configs/env.py`) — plus its
-own fields. Nothing here is a base class: the eval
-CLI, GEPA, and a trainer each declare their blocks and call these.
+own fields. Nothing here is a base class: the run CLI and a trainer each declare
+their blocks and call these.
 
 Declare the env field as `SerializeAsAny[EnvConfig] = Field(default_factory=
 single_agent_env_config)`. The `SerializeAsAny` is load-bearing: pydantic serializes

verifiers/v1/configs/client.py

@@ -75,6 +75,9 @@ class EvalClientConfig(BaseClientConfig):
     """The default (eval): forward each request to a matching endpoint via `EvalClient`."""
 
     type: Literal["eval"] = "eval"
+    cache_retention: Literal["none", "short", "long"] | None = None
+    """Anthropic prompt-cache lifetime. ``None`` follows ``PI_CACHE_RETENTION``
+    (short by default); ``long`` emits ``ttl: "1h"``; ``none`` disables markers."""
 
 
 class TrainClientConfig(BaseClientConfig):

verifiers/v1/configs/select.py

@@ -1,5 +1,5 @@
 """Which of a taskset's tasks a run uses: `Taskset.select(SelectConfig)`, under
-`--select.*` on the eval, debug, validate and GEPA CLIs (`SelectCLIConfig`)."""
+`--select.*` on the run, debug and validate CLIs (`SelectCLIConfig`)."""
 
 import re
 import sys

verifiers/v1/configs/taskset.py

@@ -17,7 +17,7 @@ class SharedTasksetConfig(BaseConfig):
     """Config passed to each task, under `--env.taskset.task.*`."""
     system_prompt: Path | None = None
     """File whose text overrides each task's `TaskData.system_prompt` on
-    iteration (e.g. a GEPA `best_system_prompt.txt`)."""
+    iteration."""
 
 
 class TasksetConfig(SharedTasksetConfig):

verifiers/v1/harnesses/pi/harness.py

@@ -26,6 +26,7 @@ logger = get_logger(__name__)
 KEY_VAR = "PI_INTERCEPT_KEY"
 
 PI_DIR = "/var/tmp/vf-pi"
+STATE_ROOT = "/tmp/vf-pi-runs"
 PACKAGES_DIR = f"{PI_DIR}/mcp"
 PI_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi"
 ACP_VERSION = "0.0.34"
@@ -104,7 +105,7 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
         data: TaskData,
     ) -> ACPConfig:
         system_prompt, prompt = self.resolve_prompt(data)
-        agent_dir = f".vf-pi-agent-{trace.id}"
+        agent_dir = f"{STATE_ROOT}/{trace.id}"
         skills_dir = f"{agent_dir}/skills"
         await self.install_skills(runtime, skills_dir)
         reasoning = ctx.sampling.reasoning_effort not in (
@@ -203,7 +204,7 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
         )
 
     async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
-        await remove_dir(runtime, f".vf-pi-agent-{trace.id}", "Pi state")
+        await remove_dir(runtime, f"{STATE_ROOT}/{trace.id}", "Pi state")
 
     async def gate_tools(
         self, config: ACPConfig, runtime: Runtime, url: str, secret: str

verifiers/v1/taskset.py

@@ -15,8 +15,8 @@ Lazy views pick which tasks an iteration yields, chainable in any order:
 
     taskset.include(idx=["0:100"]).exclude(names=["broken"]).shuffle(seed=0).take(5)
 
-`select(SelectConfig)` chains them in the config's fixed order; the eval, debug,
-validate and GEPA entrypoints and prime-rl all select through it.
+`select(SelectConfig)` chains them in the config's fixed order; the run, debug,
+and validate entrypoints and prime-rl all select through it.
 """
 
 from __future__ import annotations

verifiers/v1/utils/interrupt.py

@@ -1,4 +1,4 @@
-"""Graceful-shutdown signal handling shared across the v1 CLI entrypoints (run, gepa,
+"""Graceful-shutdown signal handling shared across the v1 CLI entrypoints (run,
 validate, debug): the first Ctrl-C/SIGTERM warns and raises KeyboardInterrupt so
 asyncio unwinds each run's teardown `finally`; further signals are swallowed so a second
 Ctrl-C can't orphan containers/sandboxes/tool-server runtimes mid-cleanup."""

verifiers/v1/utils/loaders.py

@@ -197,7 +197,7 @@ def environment_class(taskset_id: str, env_id: str = "") -> type[Env]:
 
 
 def load_environment(config: EnvConfig) -> Env:
-    """Construct the env for `config`. Every construction site (eval, serve, gepa)
+    """Construct the env for `config`. Every construction site (eval, serve)
     goes through here so subclass envs load everywhere."""
     return environment_class(config.taskset.id, config.id)(config)
 

{verifiers-0.4.2.dev34.dist-info → verifiers-0.4.2.dev44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.2.dev34
+Version: 0.4.2.dev44
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -24,7 +24,6 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: <3.15,>=3.11
 Requires-Dist: aiohttp>=3.14.1
 Requires-Dist: anthropic>=1.0.0
-Requires-Dist: gepa>=0.0.6
 Requires-Dist: h11>=0.16.0
 Requires-Dist: httpx2>=2.12.0
 Requires-Dist: httpx>=0.27.0

{verifiers-0.4.2.dev34.dist-info → verifiers-0.4.2.dev44.dist-info}/RECORD

@@ -14,14 +14,13 @@ verifiers/v1/semantic.py,sha256=nqKPPUoPL5Oh8Es_XjtcbDiQiIMKTYayXEm-EPB-jAE,5129
 verifiers/v1/session.py,sha256=Vu7YPopaiTEdmDup-aXaUIFVtPAojl5XXdcOHjrcrdI,30157
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
 verifiers/v1/task.py,sha256=zcWjBjhJrkGIL29JTnxxw6v2pbJoiek_tF5RVAVV2ZY,15919
-verifiers/v1/taskset.py,sha256=tkv74x1TMrZWDkcIKBo8XvpAo_ehynWJ5koT3QEK5zQ,9510
+verifiers/v1/taskset.py,sha256=PRwMfPtUL0Sg9pwKzJYnpLBDqJCmQ4skQPhF1oocyyw,9504
 verifiers/v1/trace.py,sha256=o7vnIt3pXRbn5JT_gYO05YaL4NlP0k8T_UlpJK3oqnA,40595
 verifiers/v1/types.py,sha256=0yhesjtRWnhgJz2cTJWWm4EaieE7ozXwH382onCHrMA,13104
 verifiers/v1/acp/__init__.py,sha256=jY7nLAbB6uzSl-f6YCz05zBH5tarBLErNTkvV0vlaho,16613
 verifiers/v1/acp/runner.py,sha256=sFsg0UgWeDjpjihejUidIV2yTapVj1Ap0cDTaEktjkI,15484
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/debug.py,sha256=baX6LRtqBNjIbmHbdaogmz3FyyItN1Z4B-TVccuYI4k,11688
-verifiers/v1/cli/gepa.py,sha256=OlO8f2mOAcZiaQVLOla5ZT36oD9XRmdDWgz4WPxNYpY,4352
 verifiers/v1/cli/output.py,sha256=vCscZ-g-sa-SLWDDQNoXBWelC_Je8lBdodvZGOe__eo,3285
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
 verifiers/v1/cli/resume.py,sha256=hqV5AxxqeIInJeEZQerQznuijhMNijFEFcHZIltfLdE,470
@@ -30,11 +29,12 @@ verifiers/v1/cli/validate.py,sha256=IXvY9Ey7snPc6pJCX9QlGO_t9KFYRwcIVj0G5TFDanU,
 verifiers/v1/clients/__init__.py,sha256=Ysig0tE_0E4Jsfgfes1XHN-fK1s_RCXdqZD6E7lK4PU,507
 verifiers/v1/clients/base.py,sha256=C0G5lRm3nxt1xmPoA2cweBgCnN2KcLepmCYq1vdDrnU,1691
 verifiers/v1/clients/client.py,sha256=HT3aQiOyYYjvz_cKEXchtUSD07-4VehebWUJL9AyON8,3166
-verifiers/v1/clients/eval.py,sha256=pWHgnRU8QQE6jnjJdwJNdvR8BeJ-z_OfjIB1tcwLpkc,8340
+verifiers/v1/clients/eval.py,sha256=GgpHZGh4IW3gC-tOZiFPp62MCL4aHik0b4afz2g3-Qo,8729
+verifiers/v1/clients/prompt_cache.py,sha256=shJhbTUwKwao6m3DsYDyHPWmhXSJJnXk3wYQhzXFsnY,11849
 verifiers/v1/clients/train.py,sha256=v1YCVNCEXw2FNQw4lpbLoq18u8TQeo9TnsYOoMWr4zE,20277
 verifiers/v1/configs/__init__.py,sha256=X7u6X7B3ieD1HZl79Tv1kg-yPHbT0QLvAebaGqxVtvA,317
 verifiers/v1/configs/agent.py,sha256=xn9j1WcBIGz7LjgW8WOWDvSvk7vPLztisSLVcT2foxk,7530
-verifiers/v1/configs/client.py,sha256=IvYQuPLVWr8qZNdMbQ9lRXK0wZ4vOjK67lO1lgD2fmM,6149
+verifiers/v1/configs/client.py,sha256=nFUI5oCih6L7by6M2l3wevghzpct8E_grmabT8lYWrA,6381
 verifiers/v1/configs/deployment.py,sha256=Eggd8phUNVcq_f56Haw-OT5U6fTBLDhBnXFJ5AEb7MI,389
 verifiers/v1/configs/env.py,sha256=DgjIbU5NxLGV7h4v-2lcDJF-y4S_VqnEqNNrNt7Ztrk,7532
 verifiers/v1/configs/harness.py,sha256=I9DcHzml17T7SYHobP7E6jHwNSL2eSqEtckioQ2pGNQ,3356
@@ -44,12 +44,12 @@ verifiers/v1/configs/retries.py,sha256=GIrwS0lnMKnlQ_Wj1Z2J6NlEYK4jX5weZxYmQmLcp
 verifiers/v1/configs/runtime.py,sha256=7To2aecnYuv_ThVt5OIlKUiocldmt4O0hZzkSwuJsBI,11773
 verifiers/v1/configs/sampling.py,sha256=bd6lietww4htrNYmUEqwK5yfKMhUT3nEjHoCg5xl_ys,1174
 verifiers/v1/configs/scoring.py,sha256=8t5NrQU9b8R9dGdcG-aQm_U1m3TgeMIbw_XUN4esoF0,3115
-verifiers/v1/configs/select.py,sha256=bLMJAb7i23IxU6BZMIS0Q_K5JYLlcwL8zi87REhrccc,5827
+verifiers/v1/configs/select.py,sha256=a72aweHXsA1B9i-ShR710NZr4ZFC-0xfSQ5J8aQGrew,5820
 verifiers/v1/configs/task.py,sha256=3CB9pumHdxAHlxLKuLEiCfQPW37tNqy_HC6DgG2GKj4,2958
-verifiers/v1/configs/taskset.py,sha256=rQow8AJL34n-ZVQ_4GPjPIW5fzBL5QU_RTjJO9tlmwU,945
+verifiers/v1/configs/taskset.py,sha256=OJG_DfM2_6I4Mm1-egMcrIEN6UtD2JaBaXnzW9SZXk8,906
 verifiers/v1/configs/cli/__init__.py,sha256=YMDUPdbRlDJ77dhj3QqzmyRhqNgUvuQdpj_bdBqzZPM,78
 verifiers/v1/configs/cli/debug.py,sha256=1vddnShdwYzRJp0Ubl9sp7sgnR3m2JKHkAOk741UnWU,2748
-verifiers/v1/configs/cli/env.py,sha256=MdN9VyGqecALu6IILYffjMXNcPcdAXvafisKq52Eajg,2584
+verifiers/v1/configs/cli/env.py,sha256=ysTSo2UDZIXS08LUrch87XLHsdTQ-emXj6QWRsLs4lk,2576
 verifiers/v1/configs/cli/run.py,sha256=j6ccPxFPiWsmCAmbZcc_EtElNOMaKS9YiiSDMqudczw,3794
 verifiers/v1/configs/cli/validate.py,sha256=j6qRi2EvacXTCARnVhkFRtX9xF3IP3eVVt-Zv2ER5Vw,4073
 verifiers/v1/dialects/__init__.py,sha256=MNOYM2NdTlSxGK4VtoEjMtD5zya2MpjPDX5jMvZUWEo,736
@@ -68,12 +68,6 @@ verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZ
 verifiers/v1/envs/single_agent/env.py,sha256=iH-AnS5-5rDZM3MaYFxzaHSVqJAaw9mQJaaJvsDMMX4,854
 verifiers/v1/envs/user_sim/__init__.py,sha256=nnT695HdqjAEouYL5wtVu6MhIp2wf5yuK-EYgxOVx0s,118
 verifiers/v1/envs/user_sim/env.py,sha256=deI4r-Utn_v4E_Vb9P_X3LzFh2GRkCO4OlQt8BOCMGY,4942
-verifiers/v1/gepa/__init__.py,sha256=6nmdRE0-34AKioPHBjTxUg5Jo_2z7tMX-OU3zMNpAJI,197
-verifiers/v1/gepa/adapter.py,sha256=J35ZBiI4N--Zgt0RuBzGZwOGxeF-nnIVEBf3rIc0Pgc,6362
-verifiers/v1/gepa/config.py,sha256=e1qYZOKzvfN6hehFSKba4hEdbR22yQEYeSEFOCqchYI,4735
-verifiers/v1/gepa/dataset.py,sha256=cS4MtQVoBOnAHKs8REQm5lKGeqUpdXPU1hROU67aBG4,2130
-verifiers/v1/gepa/reflection.py,sha256=V1VSXuszkw5Fikwqwe5KSwunhlibudZ7F8csUXraENk,1144
-verifiers/v1/gepa/runner.py,sha256=CnnOv9pvj7QCwPYDZBODUwdDEJ1yMP51bgTEm2Xxtf0,5897
 verifiers/v1/harnesses/__init__.py,sha256=SBNQmzQREsKG1ZLe1hAWdFWdXh11akXaF_9JbmJImg8,2203
 verifiers/v1/harnesses/node.py,sha256=G6hnEms726QH9KJiNZCMJUfrIrb-8hjfHkzNJP8kxMo,5160
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
@@ -107,7 +101,7 @@ verifiers/v1/harnesses/opencode/harness.py,sha256=xKVGBaQv22dRF09Bu0f3PvehgF1q0e
 verifiers/v1/harnesses/opencode/mcp-ready.js,sha256=QpAJcego5pNOZJu1dfpRLd873xitYGYwE9qwmgmvMvI,1904
 verifiers/v1/harnesses/pi/__init__.py,sha256=1mSnzOf-RdEchdeg6IHtcdobG5KIpAhuTdUIgsCIn74,117
 verifiers/v1/harnesses/pi/gate.mjs,sha256=vk_OqHXhRdYzQL7YMX8Teajm0fh9tokzDvLHVOd2I9o,449
-verifiers/v1/harnesses/pi/harness.py,sha256=NURYJCiE5kIB9CXIMYgvaCtFdiGpHaiWU84rOntF-3A,7753
+verifiers/v1/harnesses/pi/harness.py,sha256=K2FtvDhoAUU_CdKVqE_NmDwsw3vqlSaZuLJWkAX0JQE,7784
 verifiers/v1/harnesses/prime_agent/__init__.py,sha256=xfPC4RrlnyYWaf38tPAOZq4YL6M55lMpj5BoRLmaK8I,171
 verifiers/v1/harnesses/prime_agent/harness.py,sha256=sV82MCi7uZTKlOH8Vg9_WSytgZNSbknShOhmYcdPPaE,12354
 verifiers/v1/harnesses/rlm/__init__.py,sha256=hwx51xTdEWzzYgMj-p4ETNcptoub3IPMDbEdJ4uB8Ns,122
@@ -181,8 +175,8 @@ verifiers/v1/utils/format.py,sha256=zk6p_UrzIK5Rx7_lvgVuAxjlST35I9vwVDc6GCieo1o,
 verifiers/v1/utils/generic.py,sha256=lKIzu5rvwnuSlr_kTFdupf-ElubjjvOo0vnBHYMUpC4,5036
 verifiers/v1/utils/git.py,sha256=MTRinOvG4EddfjdahHkFZUihHmhJ37n5ROtgjzjE1NA,8366
 verifiers/v1/utils/image.py,sha256=OFw_wdwVtbdwa8uJ3X3vYfhpUAi0CTH5HMBupjVsRRQ,282
-verifiers/v1/utils/interrupt.py,sha256=brRlP_YHOqNlxGlj7tQmRj9R4deT_qw5EfQhm711psw,1049
-verifiers/v1/utils/loaders.py,sha256=Oq0nzTxnrfE6mKXBjD2aj4LF3MqFpK-WWrObVQQ_SQ8,13049
+verifiers/v1/utils/interrupt.py,sha256=HueVSI54ZVIKcJ0uZ62uQduUSgXEXJRL3tlzlo1FCak,1043
+verifiers/v1/utils/loaders.py,sha256=lEeRQ5j6-YJ9UKshhGKBz6g5yx6dn-M1T8urXpNP5KI,13043
 verifiers/v1/utils/logging.py,sha256=F1ymwd1A8Unu85kAQ52MAowHnR7EKQIl0RDRY8u-N5I,5392
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/network.py,sha256=SI6_PXHIAPows6VBpK75N6dMbPw6lWQQ3OjBQkSDGN4,1463
@@ -193,8 +187,8 @@ verifiers/v1/utils/scope.py,sha256=zbs4pj6AVk4jywlAJ3AX0Q0_kzB9dBB02nY-9H4ELdQ,8
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/trace_store.py,sha256=Qo9qvsJAl8PoRUavjQAfUB2VNxmUtYQtU2ejLKPzY9c,3382
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.4.2.dev34.dist-info/METADATA,sha256=Fwyk4rWxlpqu_x0GHsW8HLdS0kQwKjRuCSTRDMfZeLg,5142
-verifiers-0.4.2.dev34.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-verifiers-0.4.2.dev34.dist-info/entry_points.txt,sha256=3M8Wr0w1cUKiB-fJ6YHB6eK5WuKYg-Y3OC4wCuUje_c,174
-verifiers-0.4.2.dev34.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.4.2.dev34.dist-info/RECORD,,
+verifiers-0.4.2.dev44.dist-info/METADATA,sha256=1QsclZmVbRWmAKlpEy0XS6It5xZpqx0mfwtAzwiH7VA,5115
+verifiers-0.4.2.dev44.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+verifiers-0.4.2.dev44.dist-info/entry_points.txt,sha256=qdMg7ajiFpDsHf7tuPMRclJXizu1J6LTWawS5YS0B0I,137
+verifiers-0.4.2.dev44.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.4.2.dev44.dist-info/RECORD,,

{verifiers-0.4.2.dev34.dist-info → verifiers-0.4.2.dev44.dist-info}/entry_points.txt

@@ -1,5 +1,4 @@
 [console_scripts]
 vf-debug = verifiers.v1.cli.debug:main
-vf-gepa = verifiers.v1.cli.gepa:main
 vf-run = verifiers.v1.cli.run:main
 vf-validate = verifiers.v1.cli.validate:main

verifiers/v1/cli/gepa.py

@@ -1,104 +0,0 @@
-"""The GEPA entrypoint: `uv run vf-gepa [<taskset-id>] --model <model> [options]`.
-
-Registered as the `gepa` console script. Optimizes a v1 taskset's `Task.system_prompt` via
-GEPA (Genetic-Pareto): alternating rollouts with a teacher LM reflecting on results — see
-`verifiers.v1.gepa`. CLI resolution goes through `verifiers.v1.cli.resolve`: a
-leading bare token is the taskset id, the `env` subconfig is narrowed from the ids so
-`--env.taskset.*` / `--env.<role>.*` stay typed and `-h` renders them, `@ file.toml` loads,
-and the actual parse is `pydantic_config.cli`.
-"""
-
-import os
-import sys
-import uuid
-
-from pydantic_config import cli
-
-import verifiers.v1 as vf
-from verifiers.v1.cli.output import output_path, write_config
-from verifiers.v1.cli.resolve import (
-    extract_id,
-    narrow_config,
-    plugin_errors,
-    references_config_file,
-    with_positional_taskset,
-)
-from verifiers.v1.gepa import GEPAConfig, run_gepa
-from verifiers.v1.utils.interrupt import install_interrupt
-from verifiers.v1.utils.logging import get_logger, setup_logging
-from verifiers.v1.utils.trace_store import TRACES_FILE
-
-logger = get_logger(__name__)
-
-USAGE = "usage: uv run vf-gepa [<taskset-id>] [--env.id <id>] --model <model> [options] [@ file.toml]"
-
-
-def main(argv: list[str] | None = None) -> None:
-    # The run identity: every process this run spawns inherits it.
-    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
-    argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
-
-    if not argv or any(arg in ("-h", "--help") for arg in argv):
-        print(USAGE)
-        sys.argv = [sys.argv[0], "--help"]
-        with plugin_errors():
-            cli(
-                narrow_config(GEPAConfig, argv)
-            )  # full option help, narrowed to the given ids
-        return
-    typed_axis = any(a.startswith("--env.") for a in argv)
-    if (
-        not extract_id(argv, "env.taskset")
-        and not references_config_file(argv)
-        and not typed_axis
-    ):
-        raise SystemExit(
-            USAGE
-        )  # need a taskset (positional / --env.taskset.id) or a @ file.toml
-
-    with plugin_errors():
-        config_type = narrow_config(GEPAConfig, argv)
-        sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
-        config = cli(config_type)
-    setup_logging("DEBUG" if config.verbose else "INFO")
-    # Refuse multi-agent before the dry-run return, so --dry-run can't write a
-    # config the real invocation would reject.
-    env_cls = vf.environment_class(
-        config.env.taskset.id,
-        config.env.id,
-    )
-    if not issubclass(env_cls, vf.SingleAgentEnv):
-        raise SystemExit(
-            f"gepa: {config.env.env_id!r} runs {env_cls.__name__}, a multi-agent env; "
-            "gepa optimizes one agent's prompt against per-trace rewards and can't "
-            "drive a multi-agent interaction — only eval runs those"
-        )
-    # A named run directory is never silently reused: any write into it — the dry-run
-    # config included, which would destroy the existing traces' config provenance —
-    # would overwrite the previous run.
-    traces_file = output_path(config) / TRACES_FILE
-    if traces_file.exists() and traces_file.stat().st_size > 0:
-        raise SystemExit(
-            f"run directory {output_path(config)} already contains results - "
-            "pick another --run.name or delete it"
-        )
-
-    if config.dry_run:  # resolved + validated; write it to the output dir and exit
-        logger.info(
-            "wrote config to %s", write_config(config, output_path(config), "gepa.json")
-        )
-        return
-
-    # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out run still
-    # runs the runner's `serving()` teardown (interception / tool-server runtimes); further
-    # signals during that cleanup are swallowed so an impatient second Ctrl-C can't orphan them.
-    install_interrupt()
-
-    env = vf.load_environment(config.env)
-    try:
-        result = run_gepa(env, config)
-    except KeyboardInterrupt:
-        # Graceful cleanup already ran (run_gepa's finally tore down serving); exit on the
-        # conventional Ctrl-C code without dumping a traceback.
-        raise SystemExit(130)
-    print(f"best system prompt:\n{result.best_candidate.get('system_prompt', '')}")

verifiers/v1/gepa/__init__.py

@@ -1,5 +0,0 @@
-from verifiers.v1.gepa.adapter import GEPAAdapter
-from verifiers.v1.gepa.config import GEPAConfig
-from verifiers.v1.gepa.runner import run_gepa
-
-__all__ = ["GEPAAdapter", "GEPAConfig", "run_gepa"]

verifiers/v1/gepa/adapter.py

@@ -1,128 +0,0 @@
-"""The GEPA <-> v1 bridge: run a candidate system prompt over a batch of tasks and score it.
-
-GEPA's adapter protocol (`evaluate`, `make_reflective_dataset`) is synchronous and
-`gepa.api.optimize` blocks, but v1 rollouts are async. The runner manages one event loop by
-hand: it enters `env.serving()` on that loop and runs the blocking `optimize()` on the main
-thread, so each synchronous `evaluate()` drives its batch of rollouts with
-`loop.run_until_complete` — the one sync↔async hop. (Mirrors how v0 vf-gepa bridged; no worker
-thread, so a Ctrl-C unwinds straight through `optimize()` into the runner's teardown.)
-"""
-
-import asyncio
-from collections.abc import Awaitable, Callable, Mapping, Sequence
-from dataclasses import dataclass, field
-from typing import Any
-
-from gepa.core.adapter import EvaluationBatch
-from pydantic_core import to_jsonable_python
-
-from verifiers.v1.clients import ModelContext
-from verifiers.v1.env import Env
-from verifiers.v1.episode import Episode
-from verifiers.v1.task import Task
-
-Candidate = dict[str, str]
-
-
-@dataclass
-class GEPAAdapter:
-    """Bridges GEPA's optimization loop with a native v1 `Env`. `tasks` covers only the
-    trainset + valset tasks GEPA was given (not the whole taskset), keyed by `task.data.idx` —
-    GEPA's `batch` is a list of those idxs, and injecting the candidate rebuilds each `Task`
-    around a `data` row carrying the new `system_prompt`. `loop` is the runner's persistent event
-    loop (which holds `env.serving()` open); `evaluate` drives its rollouts on it with
-    `run_until_complete`."""
-
-    env: Env
-    ctx: ModelContext
-    tasks: dict[int, Task]
-    loop: asyncio.AbstractEventLoop
-    semaphore: asyncio.Semaphore | None = None
-    on_complete: Callable[[Episode], Awaitable[None]] | None = None
-    """Called with each rollout's episode as it finalizes — the runner's persist hook that
-    streams episodes to `traces.jsonl`."""
-    reflection_columns: list[str] = field(default_factory=list)
-    propose_new_texts: Callable[..., Candidate] | None = None
-    """Part of GEPA's adapter protocol — its proposer reads this attribute on every reflection
-    step. None = use GEPA's default reflection-LM proposer (the AttributeError from leaving it
-    undeclared silently disables all mutation proposals)."""
-
-    def evaluate(
-        self,
-        batch: list[int],
-        candidate: Candidate,
-        capture_traces: bool = False,
-    ) -> EvaluationBatch[Episode, Episode]:
-        """Run `candidate`'s system prompt on the tasks named by `batch` (`Task.idx` values)
-        and score them — one episode and one score per batch entry, so the batch stays
-        aligned however many traces the env's episode holds. Called synchronously by GEPA on
-        the main thread; each batch's rollouts run on the runner's persistent loop via
-        `run_until_complete`."""
-        system_prompt = candidate.get("system_prompt", "")
-        episodes = self.loop.run_until_complete(self._run_batch(batch, system_prompt))
-        scores = [_episode_score(episode) for episode in episodes]
-        return EvaluationBatch(
-            outputs=episodes,
-            scores=scores,
-            trajectories=episodes if capture_traces else None,
-        )
-
-    async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Episode]:
-        # Inject the candidate as a copy of each base task with its system_prompt overridden
-        # (`with_system_prompt` copies rather than reconstructs, so subclass state survives and
-        # the shared base task in `self.tasks` is left untouched for the next candidate).
-        tasks = [self.tasks[idx].with_system_prompt(system_prompt) for idx in batch]
-        slots = [slot for task in tasks for slot in self.env.slots(task)]
-        results = await asyncio.gather(
-            *(
-                self.env.run_slot(slot, self.ctx, self.semaphore, self.on_complete)
-                for slot in slots
-            )
-        )
-        return list(results)
-
-    def make_reflective_dataset(
-        self,
-        candidate: Candidate,  # Required by GEPA's adapter protocol.
-        eval_batch: EvaluationBatch[Episode, Episode],
-        components_to_update: list[str],
-    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
-        """Build the reflective dataset the teacher LM reads to propose a new system prompt,
-        from `eval_batch.trajectories` (episodes captured by a prior
-        `evaluate(capture_traces=True)` on the same batch) — one record per trace, so a
-        multi-agent episode shows the teacher every seat's turn (stamped with its role)."""
-        episodes = eval_batch.trajectories or []
-        records = []
-        for episode in episodes:
-            for trace in episode.traces:
-                record: dict[str, Any] = {
-                    "query": trace.task.data.prompt_text,
-                    "completion": trace.last_reply,
-                    "reward": trace.reward,
-                }
-                record["agent"] = trace.agent.name
-                if trace.has_error:
-                    record["error"] = str(trace.last_error)
-                if trace.stop_condition:
-                    record["stop_condition"] = trace.stop_condition
-                for column in self.reflection_columns:
-                    if column in trace.info:
-                        record[column] = to_jsonable_python(trace.info[column])
-                    elif hasattr(trace.task.data, column):
-                        record[column] = to_jsonable_python(
-                            getattr(trace.task.data, column)
-                        )
-                records.append(record)
-        return {comp: records for comp in components_to_update}
-
-
-def _episode_score(episode: Episode) -> float:
-    """A candidate's score on one episode: the mean reward of the episode's scored
-    traces. Seats that recorded no rewards (a reward-less judge) don't dilute the
-    signal; a successful episode with no scored traces scores 0."""
-    if any(trace.is_dropped for trace in episode.traces):
-        raise ValueError("GEPA cannot score an episode containing dropped traces")
-    if not episode.ok:
-        raise ValueError(f"GEPA cannot score failed episode {episode.id!r}")
-    scored = [trace.reward for trace in episode.traces if trace.rewards]
-    return sum(scored) / len(scored) if scored else 0.0

verifiers/v1/gepa/config.py

@@ -1,96 +0,0 @@
-"""The `GEPAConfig`: the single config object the `gepa` CLI parses.
-
-GEPA optimizes one taskset's `Task.system_prompt` by alternating rollouts (`evaluate`) with a
-teacher LM reflecting on the reflective dataset (`make_reflective_dataset`) — see
-`verifiers.v1.gepa.adapter.GEPAAdapter`. It owns an `env` field (the
-environment: its taskset, seats, limits) and adds the optimization loop's own knobs (model,
-reflection model, train/val split, budget). GEPA always runs in-process, since its adapter protocol is itself synchronous (see
-`GEPAAdapter`)."""
-
-from pathlib import Path
-
-from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
-from pydantic_config import BaseConfig
-
-from verifiers.v1.clients import EvalClientConfig
-from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
-from verifiers.v1.configs.cli.run import RunConfig, default_run_name
-from verifiers.v1.configs.env import EnvConfig
-from verifiers.v1.configs.sampling import SamplingConfig
-from verifiers.v1.configs.select import SelectCLIConfig
-from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
-
-
-class GEPAConfig(BaseConfig):
-    """The GEPA run plus its environment. `model` runs the rollouts under optimization;
-    `reflection_model` (defaults to `model`) proposes new system prompts from the reflective
-    dataset."""
-
-    env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig()
-    """The environment under optimization — the same `[env]` block as an eval's
-    (`--env.taskset.*`, seats, limits), narrowed to the selected env's config class."""
-
-    @model_validator(mode="before")
-    @classmethod
-    def _resolve_env(cls, data):
-        return resolve_env_field(data, narrowed_env_annotation(cls))
-
-    run: RunConfig = Field(default_factory=RunConfig)
-    """Run identity: `run.name` is the display name and `run.dir` names the directory
-    under `output_dir`; both auto-generate like an eval's when unset."""
-    model: str = Field(
-        "deepseek/deepseek-v4-flash", validation_alias=AliasChoices("model", "m")
-    )
-    """Model id for rollouts under optimization."""
-    client: EvalClientConfig = EvalClientConfig()
-    sampling: SamplingConfig = SamplingConfig()
-    reflection_model: str | None = None
-    """Teacher model that proposes new system prompts. None = reuse `model`."""
-    reflection_client: EvalClientConfig | None = None
-    """Endpoint for `reflection_model`. None = reuse `client`."""
-
-    select: SelectCLIConfig = SelectCLIConfig()
-    """Which of the taskset's tasks GEPA splits into train/val, under `--select.*`
-    (`-n` sets `select.limit`, `-s` sets `select.shuffle`)."""
-    num_train: int = Field(100, ge=1)
-    """Tasks reserved for reflection minibatches (GEPA never scores the full trainset at once)."""
-    num_val: int = Field(50, ge=1)
-    """Tasks held out to score each candidate system prompt for the pareto frontier."""
-    seed: int = 0
-    """Seed for GEPA's optimizer (candidate selection / minibatch sampling). The train/val
-    split comes from `select` (`-s` shuffles it under `select.seed`), so this doesn't
-    change it."""
-
-    max_total_rollouts: int = Field(500)
-    """Total rollouts GEPA may spend across the whole optimization run."""
-    reflection_minibatch_size: int = 3
-    """Train tasks sampled per reflection step."""
-    reflection_columns: list[str] = Field(default_factory=list)
-    """Extra per-trace fields (from `trace.info`, else `task`) to surface to the teacher LM."""
-    initial_prompt: str | None = None
-    """Seed system prompt. None = the first loaded task's `Task.system_prompt`, if any task
-    sets one (see `resolve_gepa_seed_prompt`)."""
-
-    max_concurrent: int | None = Field(
-        128, validation_alias=AliasChoices("max_concurrent", "c")
-    )
-    """Max rollouts in flight at once, across the whole run."""
-    output_dir: Path = Field(
-        Path("outputs"), validation_alias=AliasChoices("output_dir", "o")
-    )
-    """Directory that groups related runs. The run (`configs/gepa.json` + the streamed
-    `traces.jsonl`, alongside GEPA's own `candidates.json` / `run_log.json`) writes to
-    `output_dir / run.dir`."""
-    save_results: bool = True
-    verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
-    dry_run: bool = Field(False, exclude=True)
-    """Resolve + validate the config and dump it, then exit. Excluded from the
-    saved config so re-running `@ configs/gepa.json` runs for real."""
-
-    @model_validator(mode="after")
-    def auto_setup_run_name(self):
-        if self.run.name is None:
-            self.run.name = default_run_name(self.env, self.model)
-        if self.run.dir is None:
-            self.run.dir = self.run.name
-        return self

verifiers/v1/gepa/dataset.py

@@ -1,44 +0,0 @@
-"""Train/val split and upfront validation for a GEPA run.
-
-v1 tasksets have no generic train/val split concept (`TasksetConfig` has no `split` field;
-individual tasksets define ad hoc ones inconsistently), so GEPA carves one out of the tasks
-`select` yields (reproducible across runs like every other entrypoint): two
-disjoint slices.
-"""
-
-from verifiers.v1.task import Task
-
-
-def split_tasks(
-    tasks: list[Task], num_train: int, num_val: int
-) -> tuple[list[Task], list[Task]]:
-    """The selected tasks, split into disjoint `(train, val)` slices (`train` feeds reflection
-    minibatches; `val` scores each candidate for the pareto frontier)."""
-    if num_train + num_val > len(tasks):
-        raise ValueError(
-            f"requested {num_train} train + {num_val} val tasks, but the taskset only "
-            f"loaded {len(tasks)}"
-        )
-    return tasks[:num_train], tasks[num_train : num_train + num_val]
-
-
-def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> str:
-    """The system prompt GEPA starts optimizing from: `initial_prompt` if given, else the first
-    task that sets `system_prompt`. Some tasksets (e.g. `gsm8k`) bake instructions into
-    `prompt` rather than `system_prompt` and can't be optimized this way — pass `--initial-prompt`
-    to seed one explicitly.
-
-    How the resolved prompt reaches the model at rollout time — a real system message vs. folded
-    into the user prompt — is the harness's call: `Harness.resolve_prompt` owns that policy and
-    warns when it folds, so it isn't re-policed here."""
-    if initial_prompt is not None:
-        return initial_prompt
-    for task in tasks:
-        if task.data.system_prompt is not None:
-            return task.data.system_prompt
-    raise ValueError(
-        "no task in this taskset sets Task.system_prompt — some tasksets bake instructions "
-        "directly into `prompt` instead (e.g. gsm8k) and can't be optimized this way. Pass "
-        "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets "
-        "system_prompt on its task data (e.g. reverse-text, textarena)."
-    )

verifiers/v1/gepa/reflection.py

@@ -1,31 +0,0 @@
-"""The reflection (teacher) LM: GEPA's proposer calls this synchronously to turn a prompt into
-text. Backed by a vf `Judge` — `Judge.complete` owns the model client (same key / header /
-Prime-billing resolution rollouts use), so we never build one here. GEPA calls it from the
-worker thread running `optimize()`, so each call drives the async `complete` on a private loop.
-"""
-
-import asyncio
-from collections.abc import Callable
-
-from verifiers.v1.configs.judge import JudgeConfig
-from verifiers.v1.gepa.config import GEPAConfig
-from verifiers.v1.judge import Judge
-
-
-def build_reflection_lm(config: GEPAConfig) -> Callable[[str], str]:
-    client = config.reflection_client or config.client
-    judge = Judge(
-        JudgeConfig(
-            model=config.reflection_model or config.model,
-            base_url=client.base_url,
-            api_key_var=client.api_key_var,
-            headers=client.headers,
-            headers_from_env=client.headers_from_env,
-            timeout=client.timeout,
-        )
-    )
-
-    def reflection_lm(prompt: str) -> str:
-        return asyncio.run(judge.complete(prompt)).text
-
-    return reflection_lm

verifiers/v1/gepa/runner.py

@@ -1,126 +0,0 @@
-"""The GEPA runner: split tasks, drive `gepa.api.optimize` against a `GEPAAdapter`, save
-results. Unlike the async sibling runners, `optimize()` is synchronous and blocking, so it runs
-on the main thread and the runner manages one event loop by hand: `env.serving()` is entered on
-it once, each rollout batch runs via `loop.run_until_complete` (see `GEPAAdapter.evaluate`),
-and everything is torn down in `finally`. Mirrors how v0 vf-gepa bridged sync GEPA to async
-rollouts — a Ctrl-C raises on the main thread inside `optimize()` and unwinds through teardown."""
-
-import asyncio
-
-from gepa.api import optimize
-from gepa.core.result import GEPAResult
-
-from verifiers.v1.cli.output import output_path, save_config
-from verifiers.v1.clients import ModelContext
-from verifiers.v1.env import Env
-from verifiers.v1.episode import Episode
-from verifiers.v1.gepa.adapter import GEPAAdapter
-from verifiers.v1.gepa.config import GEPAConfig
-from verifiers.v1.gepa.dataset import (
-    resolve_gepa_seed_prompt,
-    split_tasks,
-)
-from verifiers.v1.gepa.reflection import build_reflection_lm
-from verifiers.v1.utils.logging import get_logger
-from verifiers.v1.utils.trace_store import append_episode
-
-logger = get_logger(__name__)
-
-
-class _GEPALog:
-    """Forwards GEPA's optimizer log lines to v1 logging. GEPA's `LoggerProtocol` needs only
-    `.log(str)`, so its per-iteration progress rides the same loguru setup as the rest of the
-    v1 CLIs."""
-
-    def log(self, message: str) -> None:
-        logger.info(message)
-
-
-def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
-    logger.info("gepa config:\n%s", config.model_dump_json(indent=2))
-    taskset = env.taskset.select(config.select)
-    all_tasks = list(taskset.take(config.num_train + config.num_val))
-    train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val)
-    selected_tasks = [*train_tasks, *val_tasks]
-    # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool —
-    # a taskset with per-task system prompts could otherwise seed from a task in neither split.
-    seed_prompt = resolve_gepa_seed_prompt(selected_tasks, config.initial_prompt)
-    tasks_by_idx = {task.data.idx: task for task in selected_tasks}
-
-    run_dir = output_path(config) if config.save_results else None
-    if run_dir is not None:
-        save_config(
-            config, run_dir, "gepa.json"
-        )  # resolved config + a fresh traces.jsonl
-        logger.info("results: %s", run_dir)
-
-    # optimize() is synchronous and blocking, so it drives the run from this (main) thread. We
-    # own one event loop: `env.serving()` (shared tool servers + interception pool, built
-    # once) is entered on it, each rollout batch runs on it via `loop.run_until_complete`
-    # (GEPAAdapter.evaluate), and it's all torn down in `finally`. Keeping optimize() on the
-    # main thread means a Ctrl-C raises straight through it into this teardown.
-    loop = asyncio.new_event_loop()
-    semaphore = (
-        asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
-    )
-    # Stream every rollout's episode to traces.jsonl as it finalizes — the same persist hook
-    # `env.run_slot` takes (each trace records its candidate prompt).
-    write_lock = asyncio.Lock()
-
-    async def on_complete(episode: Episode) -> None:
-        if run_dir is not None:
-            await append_episode(run_dir, episode, write_lock)
-
-    try:
-        # The endpoint stays config: the interception server builds the live client.
-        ctx = ModelContext(
-            client=config.client, model=config.model, sampling=config.sampling
-        )
-        reflection_lm = build_reflection_lm(config)
-        serving = env.serving()
-        loop.run_until_complete(serving.__aenter__())
-        # Pair __aexit__ with a *successful* __aenter__: this inner block is reached only once
-        # serving is up, so a startup failure propagates as-is instead of being masked by
-        # tearing down a context that never entered.
-        try:
-            adapter = GEPAAdapter(
-                env=env,
-                ctx=ctx,
-                tasks=tasks_by_idx,
-                loop=loop,
-                semaphore=semaphore,
-                on_complete=on_complete,
-                reflection_columns=config.reflection_columns,
-            )
-            optimize_kwargs: dict = {
-                "seed_candidate": {"system_prompt": seed_prompt},
-                "trainset": [task.data.idx for task in train_tasks],
-                "valset": [task.data.idx for task in val_tasks],
-                "adapter": adapter,
-                "reflection_lm": reflection_lm,
-                "max_metric_calls": config.max_total_rollouts,
-                "reflection_minibatch_size": config.reflection_minibatch_size,
-                "run_dir": str(run_dir) if run_dir is not None else None,
-                "seed": config.seed,
-                "display_progress_bar": False,
-                "skip_perfect_score": False,
-                "logger": _GEPALog(),
-            }
-            result = optimize(**optimize_kwargs)
-            if run_dir is not None:
-                # Persist the winning prompt as a plain file so it can be handed straight to
-                # eval/train via `--env.taskset.system-prompt` (see TasksetConfig).
-                candidate = result.best_candidate
-                best = (
-                    candidate.get("system_prompt", "")
-                    if isinstance(candidate, dict)
-                    else str(candidate)
-                )
-                best_path = run_dir / "best_system_prompt.txt"
-                best_path.write_text(best, encoding="utf-8")
-                logger.info("best system prompt: %s", best_path)
-            return result
-        finally:
-            loop.run_until_complete(serving.__aexit__(None, None, None))
-    finally:
-        loop.close()