verifiers 0.2.2.dev49__py3-none-any.whl → 0.2.2.dev50__py3-none-any.whl

verifiers/v1/configs/judge.py

@@ -5,7 +5,7 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
-from pydantic import BaseModel, SerializeAsAny, model_validator
+from pydantic import BaseModel, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
@@ -20,15 +20,8 @@ class JudgeConfig(BaseClientConfig):
     weight: float = 1.0
     model: str = "openai/gpt-5.4-nano"
     sampling: SamplingConfig = SamplingConfig()
-    prompt: str | None = None
-    prompt_file: Path | None = None
-    """Prompt file override, mutually exclusive with `prompt`."""
-
-    @model_validator(mode="after")
-    def check_prompt_source(self) -> "JudgeConfig":
-        if self.prompt is not None and self.prompt_file is not None:
-            raise ValueError("set `prompt` or `prompt_file`, not both")
-        return self
+    prompt: Path | None = None
+    """File whose text overrides the judge's default prompt template."""
 
 
 Judges = list[SerializeAsAny[JudgeConfig]]

verifiers/v1/configs/taskset.py

@@ -1,5 +1,7 @@
 """The taskset plugin's config: which rows load, under `--env.taskset.*`."""
 
+from pathlib import Path
+
 from pydantic import SerializeAsAny
 from pydantic_config import BaseConfig
 
@@ -14,6 +16,9 @@ class TasksetConfig(BaseConfig):
     positional `eval <taskset-id>`)."""
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
+    system_prompt: Path | None = None
+    """File whose text overrides each task's `TaskData.system_prompt` in
+    `Taskset.select` (e.g. a GEPA `best_system_prompt.txt`)."""
 
     @property
     def name(self) -> str:

verifiers/v1/envs/agentic_judge/env.py

@@ -180,41 +180,31 @@ class JudgeTask(vf.Task):
 class JudgeTaskConfig(vf.BaseConfig):
     """The judge's minted task: the grading policy and what lands in its box."""
 
-    prompt: Path | str | None = None
-    """Grading-policy override: inline text, or a policy file (a value ending in
-    `.md`/`.txt` is read from disk). Replaces only the policy body — the verdict
-    contract and workspace note are always appended, so a custom policy cannot
-    break verdict scraping. May reference `{prompt}` (the solver task's prompt);
-    if it doesn't, the task statement is appended after the policy."""
-    hint: Path | str | None = None
-    """Optional hints injected as their own section (inline text or a `.md`/
-    `.txt` file): task-family pointers into the trace or box — e.g. for math,
-    where the reference answer lives in the record; for SWE, to diff the repo
-    or read `info.patch`."""
+    prompt: Path | None = None
+    """Grading-policy file. Replaces only the policy body — the verdict contract
+    and workspace note are always appended. May reference `{prompt}` (the solver
+    task's prompt); if it doesn't, the task statement is appended after."""
+    hint: Path | None = None
+    """Optional hints file injected as their own section: task-family pointers
+    into the trace or box — e.g. for math, where the reference answer lives in
+    the record; for SWE, to diff the repo or read `info.patch`."""
     rubric: Path | None = None
     """Criteria the judge grades against: a `.toml`/`.json` file with a
     `criteria` list — the plugged rubric judge's format, so the same rubric
     files work for both. None grades the single built-in `solved` criterion."""
 
-    @staticmethod
-    def _resolve(value: Path | str) -> str:
-        path = Path(value)
-        if isinstance(value, Path) or path.suffix in (".md", ".txt"):
-            return path.read_text(encoding="utf-8")
-        return str(value)
-
     def build_prompt(self) -> str:
         if self.prompt is None:
             return GRADE_PROMPT + "\n\n" + TASK_SECTION
-        return self._resolve(self.prompt)
+        return self.prompt.read_text()
 
     def build_hint(self) -> str | None:
-        return self._resolve(self.hint) if self.hint is not None else None
+        return self.hint.read_text() if self.hint is not None else None
 
     def criteria(self) -> list[Criterion]:
         if self.rubric is None:
             return [SOLVED]
-        text = self.rubric.read_text(encoding="utf-8")
+        text = self.rubric.read_text()
         data = (
             tomllib.loads(text)
             if self.rubric.suffix.lower() == ".toml"

verifiers/v1/gepa/adapter.py

@@ -68,14 +68,10 @@ class GEPAAdapter:
         )
 
     async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Episode]:
-        # Inject the candidate by rebuilding each Task around a data row with the new
-        # system_prompt (TaskData is frozen; behavior/config carry over unchanged).
-        tasks = [
-            type(t)(
-                t.data.model_copy(update={"system_prompt": system_prompt}), t.config
-            )
-            for t in (self.tasks[idx] for idx in batch)
-        ]
+        # Inject the candidate as a copy of each base task with its system_prompt overridden
+        # (`with_system_prompt` copies rather than reconstructs, so subclass state survives and
+        # the shared base task in `self.tasks` is left untouched for the next candidate).
+        tasks = [self.tasks[idx].with_system_prompt(system_prompt) for idx in batch]
         slots = [slot for task in tasks for slot in self.env.slots(task)]
         results = await asyncio.gather(
             *(

verifiers/v1/gepa/runner.py

@@ -105,7 +105,20 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
                 "skip_perfect_score": False,
                 "logger": _GEPALog(),
             }
-            return optimize(**optimize_kwargs)
+            result = optimize(**optimize_kwargs)
+            if run_dir is not None:
+                # Persist the winning prompt as a plain file so it can be handed straight to
+                # eval/train via `--env.taskset.system-prompt` (see TasksetConfig).
+                candidate = result.best_candidate
+                best = (
+                    candidate.get("system_prompt", "")
+                    if isinstance(candidate, dict)
+                    else str(candidate)
+                )
+                best_path = run_dir / "best_system_prompt.txt"
+                best_path.write_text(best, encoding="utf-8")
+                logger.info("best system prompt: %s", best_path)
+            return result
         finally:
             loop.run_until_complete(serving.__aexit__(None, None, None))
     finally:

verifiers/v1/judge.py

@@ -116,13 +116,13 @@ def judge_config_cls(cls: type) -> type[JudgeConfig]:
 
 class Judge(Generic[ParsedT, ConfigT]):
     prompt: str | None = None
-    """Default prompt template, overridden by config."""
+    """Default prompt template, overridden by a config `prompt` file."""
     schema: type[BaseModel] | None = None
 
     def __init__(self, config: ConfigT | None = None) -> None:
         self.config = cast(ConfigT, config or judge_config_cls(type(self))())
-        if self.config.prompt_file is not None:
-            self.prompt = self.config.prompt_file.read_text(encoding="utf-8")
+        if self.config.prompt is not None:
+            self.prompt = self.config.prompt.read_text()
 
     @property
     def reward_name(self) -> str:
@@ -132,7 +132,7 @@ class Judge(Generic[ParsedT, ConfigT]):
         return judge_key(self.config) or fallback or "judge"
 
     def build_messages(self, **fields: Any) -> str | Messages:
-        template = self.config.prompt or self.prompt
+        template = self.prompt
         if template is None:
             raise ValueError(
                 f"{type(self).__name__} has no `prompt`; set it or override build_messages"

verifiers/v1/task.py

@@ -24,10 +24,11 @@ wraps it in the declared `Task` — one task type per taskset.
 
 from __future__ import annotations
 
+import copy
 import inspect
 import logging
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, ClassVar, Generic
+from typing import TYPE_CHECKING, ClassVar, Generic, Self
 
 from pydantic import ConfigDict, Field
 from pydantic_config import BaseConfig
@@ -197,6 +198,15 @@ class Task(Generic[DataT, StateT, ConfigT]):
         self.data = data
         self.config = config if config is not None else task_config_cls(type(self))()
 
+    def with_system_prompt(self, system_prompt: str) -> Self:
+        """A shallow copy of this task with `data.system_prompt` overridden. Copies the
+        instance instead of reconstructing via `type(self)(...)`, so a subclass with a
+        non-`(data, config)` constructor or extra load-time state keeps it. Used to apply the
+        config-layer / GEPA system prompt (see `TasksetConfig` and `verifiers.v1.gepa`)."""
+        clone = copy.copy(self)
+        clone.data = self.data.model_copy(update={"system_prompt": system_prompt})
+        return clone
+
     def plugged_judges(self) -> list[Judge]:
         from verifiers.v1.loaders import load_judge
 

verifiers/v1/taskset.py

@@ -79,10 +79,15 @@ class Taskset(Generic[TaskT, TasksetConfigT]):
                     "taking the first %d generated tasks",
                     num_tasks,
                 )
-            return list(itertools.islice(self.load(), num_tasks))
+            shuffle = False  # can't materialize the whole taskset to sample from
         if shuffle:
-            return sample(self.load(), shuffle=True, limit=num_tasks)
-        return list(itertools.islice(self.load(), num_tasks))
+            tasks = sample(self.load(), shuffle=True, limit=num_tasks)
+        else:
+            tasks = list(itertools.islice(self.load(), num_tasks))
+        if self.config.system_prompt is not None:
+            override = self.config.system_prompt.read_text()
+            tasks = [t.with_system_prompt(override) for t in tasks]
+        return tasks
 
     def server_config(self, server_cls: type) -> BaseConfig:
         """The config a `tools` entry is built with, resolved off `self.config` (the

verifiers/v1/tasksets/lean/taskset.py

@@ -61,7 +61,6 @@ class LeanTaskConfig(TaskConfig):
 class LeanConfig(TasksetConfig):
     dataset: LeanDatasetConfig
     docker_image: str = DEFAULT_DOCKER_IMAGE
-    system_prompt: str = DEFAULT_SYSTEM_PROMPT
     task: LeanTaskConfig = LeanTaskConfig()
 
 
@@ -187,7 +186,7 @@ class LeanTaskset(Taskset[LeanTask, LeanConfig]):
                     idx=index,
                     name=str(name) if name else f"task_{index:05d}",
                     prompt=self._build_prompt(formal_statement, header),
-                    system_prompt=config.system_prompt,
+                    system_prompt=DEFAULT_SYSTEM_PROMPT,
                     image=config.docker_image,
                     workdir=config.task.lean_project_path,
                     resources=resources,

{verifiers-0.2.2.dev49.dist-info → verifiers-0.2.2.dev50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev49
+Version: 0.2.2.dev50
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev49.dist-info → verifiers-0.2.2.dev50.dist-info}/RECORD

@@ -175,7 +175,7 @@ verifiers/v1/episode.py,sha256=KWyM9ovTphEIo2260BYu2HvtnOOTxjLRDPrF_jTCiuA,2497
 verifiers/v1/errors.py,sha256=slZnrtqMo_BgXQV46wJJhV6vvAAnVsuCbVGp2vBVWfs,6888
 verifiers/v1/graph.py,sha256=-j8MNdIv1r5ifajv5ggnGPsfBNB74IO3WKLcrpYh9hg,29102
 verifiers/v1/harness.py,sha256=cDiDuhp29aqL4CxiqSZz-IT0QlOcVlye_vTTHZblWo8,10948
-verifiers/v1/judge.py,sha256=VnTi9iSfCJMCccYWc1qCTcT7iD3AcomS2cXfHwh1HVw,9423
+verifiers/v1/judge.py,sha256=UmBcEmyY30Zod-Nnvj8UTSyvee5NAjBJE9ufPrwTiTw,9391
 verifiers/v1/legacy.py,sha256=nkbdhFJZN1QzsjE42AB_A_c1zrk3_Y8zR07srZJS2ck,22584
 verifiers/v1/loaders.py,sha256=TnNKY2msVo3p7m-5wjw376PYH-zTu3TLixXRst0VOiM,9985
 verifiers/v1/push.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtvI,11363
@@ -184,8 +184,8 @@ verifiers/v1/rollout.py,sha256=GJzHgJFlw8O1vDtWM__2JhSiG2GqbfjtH2ri1MRfw3g,20495
 verifiers/v1/scoring.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/session.py,sha256=J6ZPuvsg6vMTJAs9dMUMtfdjCuvp6Hom877DVdKFKlY,6912
 verifiers/v1/state.py,sha256=R8tyQv8nsFV2pztrquDqCOa1Mk1fAp3w2GjqUugZwE0,689
-verifiers/v1/task.py,sha256=ad2YgS7ZeeBdezyWpuexLeGsCyIk5uuUJ061LXrYJ88,11074
-verifiers/v1/taskset.py,sha256=PX1-skAVhSpGF07qjxcG0Ctmq1LiMftMxxok-uc8qRY,3939
+verifiers/v1/task.py,sha256=tmsxn1YLsgIIRrCqueBXvoupXhFks61_GNiMVTs495k,11652
+verifiers/v1/taskset.py,sha256=gkDqK_1SEU-oyDaqcv1fykyTZc9Hlub_Npe6bajhmPY,4175
 verifiers/v1/trace.py,sha256=9KqBeGaRUtWh6Gpdf6G1c7UZso0DB9Mjmtkzl6I_oYE,19458
 verifiers/v1/types.py,sha256=5tZyG4r4bLJ9a14oybHy7bSCpt5X5A17eRwmIszuRo8,9118
 verifiers/v1/acp/__init__.py,sha256=9dwH6fLopNndRmcpRSYC8ghzzMquaLfgh645QM-Dwic,2189
@@ -217,12 +217,12 @@ verifiers/v1/configs/__init__.py,sha256=X7u6X7B3ieD1HZl79Tv1kg-yPHbT0QLvAebaGqxV
 verifiers/v1/configs/agent.py,sha256=N2Lm0iZXFxfsSIiXUQ7f3RDZcoin3M7pJj6HVdCE5eo,3537
 verifiers/v1/configs/env.py,sha256=4aUap3BJX3WCA2_fZ-jGPxF3khiVpORqmz_3eBT4j9w,7824
 verifiers/v1/configs/harness.py,sha256=CG8hGbHl3rr6yawH3YYMAFl01M7Piiwf4w5oiw2Vagg,1564
-verifiers/v1/configs/judge.py,sha256=ZJlmWogHY8oDjvWvtNuVKvkzuheWalIjl1njJ4dE0Mk,2511
+verifiers/v1/configs/judge.py,sha256=eEVtBGjAOmm1sSRd_UHVBpJEfUxMHxHDBFnvchmxs-w,2217
 verifiers/v1/configs/legacy.py,sha256=YOJM6L5u6-jYpZ1CLpPInufKdmz1lEsMNKIRdT2JaaY,1080
 verifiers/v1/configs/retries.py,sha256=-nPlgmz7J_NZ4GOYG6oCc3thXo4La2SufBmvQGvvPGo,821
 verifiers/v1/configs/serve.py,sha256=cHHnFGQtqEwvaovdeoVMpCebTHgPXKUuIivN5IhxZR0,2596
 verifiers/v1/configs/task.py,sha256=1ozZjfpahin7I_TqzC7f3xr7CHSthAdydegFPXW4d9k,1101
-verifiers/v1/configs/taskset.py,sha256=IdZwsarn0_WvYdR_Kr4QYYdqbaoGYkbYyO5VA-RPYPc,657
+verifiers/v1/configs/taskset.py,sha256=-WfRRRDvLwV4v4QoBtHegYNAHExM1nCR7TZjJyNv_aA,858
 verifiers/v1/configs/cli/__init__.py,sha256=3wBAONw5XkBPqbMjpHk5XBHlBVybR8JQ2JCPWijYBMU,444
 verifiers/v1/configs/cli/debug.py,sha256=3lkKqmQFLJBcUZECChBHCrod5pbCKSmT47EhjILrxjI,2843
 verifiers/v1/configs/cli/env.py,sha256=XgjRXYpPjga0ZYIHEESY7VuVvtE6pWfAzf6JBLeuTIc,2695
@@ -238,7 +238,7 @@ verifiers/v1/dialects/chat.py,sha256=R_otqi5N0snB1QVKIYloNeLHR9kc7OmI06HYg1ir_Lw
 verifiers/v1/dialects/responses.py,sha256=ismnxUikPcPUIm80FUTPBbj-Cn1e0yGAsISMEseAWRc,13679
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=X7vQbbbfmJ_j-mJEfBkuB4a8QUPkUYkpG7yp0xYOEmg,279
-verifiers/v1/envs/agentic_judge/env.py,sha256=3XhGiMdKJBhpSyUdlfYYKPIR42Ll34WUvEFqbCtKdX8,15342
+verifiers/v1/envs/agentic_judge/env.py,sha256=2g9yz3QfYUc_OZBkxzWOEKsGiT51siYMKB7PwvbCY2E,14878
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
@@ -246,11 +246,11 @@ verifiers/v1/envs/single_agent/env.py,sha256=lPs_a6VA7KoH1xqA3T8S32Jzb_d4zrx0_54
 verifiers/v1/envs/user_sim/__init__.py,sha256=nnT695HdqjAEouYL5wtVu6MhIp2wf5yuK-EYgxOVx0s,118
 verifiers/v1/envs/user_sim/env.py,sha256=9L_KXjzfkTnDeQpn4ftQagZyPkJ3yjcQeZqZaq7kxjE,4567
 verifiers/v1/gepa/__init__.py,sha256=6nmdRE0-34AKioPHBjTxUg5Jo_2z7tMX-OU3zMNpAJI,197
-verifiers/v1/gepa/adapter.py,sha256=my7hM6GSzOw__F7Ot1E-XA_OhZyovhgbJ7fOQkW66l8,6149
+verifiers/v1/gepa/adapter.py,sha256=YNvHMR2L5Utl-vtPfjZxmDCd2tTcoAPWG6aVGnV1icM,6139
 verifiers/v1/gepa/config.py,sha256=gFn8Re4GoeHJs2PZ-iw4jCEZ72AiM1lNgwaA0uvJlYw,4626
 verifiers/v1/gepa/dataset.py,sha256=N2QYqUljHVuAdCbUuydEIlarvAWa6LFdGKu9qsOvjro,2186
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
-verifiers/v1/gepa/runner.py,sha256=o7Am2Ram-Ejbr_tVpN70zb0uQ5a1NIpXvlMhwWL22Xs,5406
+verifiers/v1/gepa/runner.py,sha256=6iJj2HFiC-NZbontq7P9lzzyeq6B0sRBz6trhxrkIW0,6075
 verifiers/v1/harnesses/__init__.py,sha256=3fgfAMzFrNgu7RIO_evZPxipPdiskiHun5EKsdS4y6k,1301
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
 verifiers/v1/harnesses/bash/harness.py,sha256=ZtjhsANuK3vXkf9mRCi8-sBMyPebJMBNHa58vTd2Fm8,5284
@@ -311,7 +311,7 @@ verifiers/v1/tasksets/harbor/__init__.py,sha256=wwVGA8N7E2Q2qA5WGHuLRrn4DgG5oaRH
 verifiers/v1/tasksets/harbor/taskset.py,sha256=2Iw1ZaPRhRjdbfClRLd0iqKWABjM7M525zFN4wuaeCk,13867
 verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
 verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
-verifiers/v1/tasksets/lean/taskset.py,sha256=0KujQgx2fTL_DyWSQidebKoD94U8oJp82PXoco6-3B4,9065
+verifiers/v1/tasksets/lean/taskset.py,sha256=Fcn4UgTcdjMYnJ4CThaFZ28Rz_QDx0e21vJdW_vp12g,9019
 verifiers/v1/tasksets/openenv/__init__.py,sha256=G726meYYDplC3QE3AyjdSShq7FUcb4NkVqowuwPPuJk,303
 verifiers/v1/tasksets/openenv/taskset.py,sha256=1Wc9-5eN2HMKAwL34yTNex3zYOvXNXibF5wCROoMbFI,6075
 verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2LtcI7VDUHYPzRE,329
@@ -329,8 +329,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=JczGzBn6s3wsIrSY2Hy3hE8m6NreNgsNzhSjDikQqX0,1037
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev49.dist-info/METADATA,sha256=58Q74xtCzA2vwqY2VjgOgFdSvyqzrV2vAyCCkgjOrCo,4545
-verifiers-0.2.2.dev49.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev49.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev49.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev49.dist-info/RECORD,,
+verifiers-0.2.2.dev50.dist-info/METADATA,sha256=CmQlOqkHtzYaf_70Po3zu37d6_rPDPvoKZUZXjLBsfs,4545
+verifiers-0.2.2.dev50.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev50.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev50.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev50.dist-info/RECORD,,