verifiers 0.3.2.dev127__py3-none-any.whl → 0.3.2.dev129__py3-none-any.whl

verifiers/v1/cli/debug.py

@@ -3,10 +3,12 @@
 import asyncio
 import contextlib
 import logging
+import os
 import shlex
 import sys
 import time
 import traceback
+import uuid
 from collections.abc import Awaitable
 from pathlib import Path
 from typing import Any
@@ -316,6 +318,8 @@ async def run_debug(config: DebugConfig) -> list[Trace]:
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = with_positional_taskset(
         list(sys.argv[1:]) if argv is None else list(argv), flag="--taskset.id"
     )

verifiers/v1/cli/eval/runner.py

@@ -9,6 +9,7 @@ runs env servers); this CLI is the quick local path.
 import asyncio
 import contextlib
 import logging
+import os
 import time
 from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
 from typing import TypeVar, cast
@@ -139,6 +140,8 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
 
     # Opened before the first rollout so every episode streams as it lands.
     run = open_run(config, push_state, num_examples=len(tasks))
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", config.run.id)
     # Resumed rollouts are part of this run too.
     log_episodes(run, finished)
 

verifiers/v1/cli/gepa.py

@@ -9,7 +9,9 @@ and the actual parse is `pydantic_config.cli`.
 """
 
 import logging
+import os
 import sys
+import uuid
 
 from pydantic_config import cli
 
@@ -32,6 +34,8 @@ USAGE = "usage: uv run vf-gepa [<taskset-id>] [--env.id <id>] --model <model> [o
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
 
     if not argv or any(arg in ("-h", "--help") for arg in argv):

verifiers/v1/cli/replay.py

@@ -12,8 +12,10 @@ import asyncio
 import contextlib
 import json
 import logging
+import os
 import sys
 import time
+import uuid
 from pathlib import Path
 
 from pydantic_config import cli
@@ -197,6 +199,8 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = list(sys.argv[1:]) if argv is None else list(argv)
     if not argv or any(a in ("-h", "--help") for a in argv):
         print(USAGE)

verifiers/v1/cli/validate.py

@@ -4,9 +4,11 @@ import asyncio
 import contextlib
 import json
 import logging
+import os
 import shutil
 import sys
 import time
+import uuid
 from collections import Counter, defaultdict
 from collections.abc import Mapping, Sequence
 from pathlib import Path
@@ -425,6 +427,8 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = with_positional_taskset(
         list(sys.argv[1:]) if argv is None else list(argv), flag="--taskset.id"
     )

verifiers/v1/gepa/dataset.py

@@ -40,5 +40,5 @@ def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> s
         "no task in this taskset sets Task.system_prompt — some tasksets bake instructions "
         "directly into `prompt` instead (e.g. gsm8k) and can't be optimized this way. Pass "
         "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets "
-        "system_prompt on its task data (e.g. reverse-text, lean, textarena)."
+        "system_prompt on its task data (e.g. reverse-text, textarena)."
     )

verifiers/v1/interception/tunnel/prime.py

@@ -9,15 +9,19 @@ from collections.abc import AsyncIterator
 from typing import Literal
 
 from verifiers.v1.interception.tunnel.base import BaseTunnelConfig, Tunnel
-from verifiers.v1.runtimes.limiters import creation_limiter
+from verifiers.v1.runtimes.limiters import CreationLimiter
 from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.prime import ensure_prime_auth
+from verifiers.v1.utils.scope import run_scope
 
 # The prime_tunnel service caps tunnel starts at 512/min per API token — a property of the
-# tunnel service, shared by every process for the user that opens one. One user-global
+# tunnel service, shared by every process of a run that opens one. One run-scoped
 # limiter, not a per-runtime config knob.
 _TUNNELS_PER_MIN = 512
-TUNNEL_LIMITER = creation_limiter(_TUNNELS_PER_MIN / 60, "prime-tunnel")
+
+
+def tunnel_limiter() -> CreationLimiter:
+    return CreationLimiter("prime-tunnel", run_scope(), _TUNNELS_PER_MIN / 60)
 
 
 class PrimeTunnelConfig(BaseTunnelConfig):
@@ -35,8 +39,8 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
     @contextlib.asynccontextmanager
     async def expose(self, port: int) -> AsyncIterator[str]:
         """Bridge the host `port` to a public URL via prime_tunnel (frpc). Tunnel creation
-        is network-bound and globally rate-capped (512/min, user-wide via the shared
-        `TUNNEL_LIMITER`), so transient failures are retried; a terminal one raises
+        is network-bound and rate-capped (512/min, run-wide via the shared
+        `tunnel_limiter`), so transient failures are retried; a terminal one raises
         `TunnelError`. The tunnel is torn down on exit."""
         from prime_tunnel import Tunnel as TunnelClient
 
@@ -48,7 +52,7 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
             async for attempt in retrying(retries=3, label=label):
                 with attempt:
                     client = TunnelClient(local_port=port)
-                    async with TUNNEL_LIMITER:
+                    async with tunnel_limiter():
                         url = str(await client.start()).rstrip("/")
         except Exception as e:
             raise TunnelError(f"{label} failed: {e}") from e

verifiers/v1/runtimes/limiters.py

@@ -1,11 +1,12 @@
-"""User-global creation-rate limiters for the remote runtimes.
+"""Creation-rate limiters for the remote runtimes.
 
 A leaky bucket backed by a lock file under the user cache (``~/.cache/verifiers``, falling
-back to the temp dir when no home is resolvable), so a provider's per-account creation rate
-(Modal sandboxes, Prime tunnels) is enforced across EVERY process for the user —
-the single-process eval and all the elastically-spawned env-server worker processes alike — not
-just within one process. Keyed by name: one bucket file per name, shared by every process (and
-run) for the user.
+back to the temp dir when no home is resolvable), so a provider's creation rate (Modal
+sandboxes, Prime sandboxes and tunnels) is enforced across EVERY process that shares the
+bucket — the eval process and all the elastically-spawned env-server worker processes alike —
+not just within one process. A bucket is named by the limiter's name and a scope the caller
+picks; the runtimes pass the run id, so one run's backlog (or the reservations a killed run
+left behind) never delays another run.
 """
 
 import asyncio
@@ -22,17 +23,18 @@ LIMITER_DIR = CACHE_DIR / "limiter"
 class CreationLimiter:
     """An async leaky bucket shared across processes via a lock file: each `async with`
     reserves the next `1/per_sec`-spaced slot (advancing the on-disk cursor under an exclusive
-    flock) and sleeps until it, so the aggregate creation rate across all of the user's
-    processes stays at `per_sec`. The reservation runs off the event loop; the wait does not
-    hold the lock. Backlogs over five minutes fail rather than silently stalling creation."""
+    flock) and sleeps until it, so the aggregate creation rate across every process sharing
+    the bucket stays at `per_sec`. The reservation runs off the event loop; the wait does not
+    hold the lock. Reservations are never released, so a cancelled waiter still holds its
+    slot; the backlog drains at `per_sec` regardless."""
 
-    def __init__(self, name: str, per_sec: float) -> None:
+    def __init__(self, name: str, scope: str, per_sec: float) -> None:
         self._interval = 1 / per_sec
-        self._path = LIMITER_DIR / f"{name}.bucket"
+        self._path = LIMITER_DIR / f"{name}-{scope.replace('/', '--')}.bucket"
 
     def _reserve(self) -> float:
         os.makedirs(LIMITER_DIR, exist_ok=True)
-        # Shared buckets require a clock comparable across hosts and boots.
+        # Shared buckets require a clock comparable across the run's hosts.
         with open(self._path, "a+") as f:
             fcntl.flock(f.fileno(), fcntl.LOCK_EX)
             try:
@@ -40,19 +42,13 @@ class CreationLimiter:
                 data = f.read().strip()
                 now = time.time()
                 slot = max(now, float(data) if data else 0.0)
-                wait = slot - now
-                if wait > 5 * 60:
-                    raise TimeoutError(
-                        f"{self._path.stem} creation limiter backlog of {wait:.1f}s "
-                        f"exceeds 300s ({self._path})"
-                    )
                 f.seek(0)
                 f.truncate()
                 f.write(repr(slot + self._interval))
                 f.flush()
-                return wait
             finally:
                 fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+        return slot - now
 
     async def __aenter__(self) -> Self:
         wait = await asyncio.to_thread(self._reserve)
@@ -64,16 +60,12 @@ class CreationLimiter:
         return False
 
 
-_creation_limiters: dict[str, CreationLimiter] = {}
-
-
-def creation_limiter(per_sec: float | None, name: str) -> CreationLimiter | None:
-    """A user-global limiter pacing `name`'s creation to `per_sec`/s (None/<= 0 disables).
-
-    All callers (and processes) sharing a `name` share one bucket, so use one rate per name."""
+def creation_limiter(
+    per_sec: float | None, name: str, scope: str
+) -> CreationLimiter | None:
+    """A limiter pacing `name`'s creation to `per_sec`/s within `scope` (None/<= 0
+    disables). All callers (and processes) sharing a name and scope share one bucket, so
+    use one rate per name."""
     if not per_sec or per_sec <= 0:
         return None
-    limiter = _creation_limiters.get(name)
-    if limiter is None:
-        limiter = _creation_limiters[name] = CreationLimiter(name, per_sec)
-    return limiter
+    return CreationLimiter(name, scope, per_sec)

verifiers/v1/runtimes/modal.py

@@ -31,6 +31,7 @@ from verifiers.v1.runtimes.base import (
 )
 from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.scope import run_scope
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +94,7 @@ class ModalConfig(NetworkPolicyConfig):
     """Disk in GB. Modal sandboxes have no disk knob, so this is accepted (so a task can
     declare it without a warning) but not enforced."""
     creates_per_sec: float | None = 40.0
-    """Pace sandbox creation to this many per second, enforced user-wide across every
+    """Pace sandbox creation to this many per second, enforced run-wide across every
     env-server worker process (None/<= 0 disables it)."""
 
     @model_validator(mode="after")
@@ -181,7 +182,9 @@ class ModalRuntime(Runtime):
         try:
             app = await modal.App.lookup.aio(_APP_NAME, create_if_missing=True)
             async with (
-                creation_limiter(self.config.creates_per_sec, "modal-sandbox")
+                creation_limiter(
+                    self.config.creates_per_sec, "modal-sandbox", run_scope()
+                )
                 or contextlib.nullcontext()
             ):
                 await run_shielded(self._create_sandbox(app))

verifiers/v1/runtimes/prime.py

@@ -28,6 +28,7 @@ from verifiers.v1.runtimes.base import (
 from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.prime import ensure_prime_auth
+from verifiers.v1.utils.scope import run_scope
 
 logger = logging.getLogger(__name__)
 
@@ -87,9 +88,9 @@ class PrimeConfig(NetworkPolicyConfig):
     idle_timeout: float | None = 3600
     """Seconds of inactivity before the sandbox self-deletes (None disables)."""
     creates_per_min: int | None = None
-    """Pace sandbox creation to this many per minute, enforced user-wide across every
+    """Pace sandbox creation to this many per minute, enforced run-wide across every
     env-server worker process (None/<= 0 disables it). (Tunnel creation is limited separately
-    and globally — see interception.tunnel.prime.TUNNEL_LIMITER.)"""
+    — see interception.tunnel.prime.tunnel_limiter.)"""
 
     @model_validator(mode="after")
     def _validate_egress(self) -> "PrimeConfig":
@@ -175,7 +176,9 @@ class PrimeRuntime(Runtime):
         try:
             async with (
                 creation_limiter(
-                    (self.config.creates_per_min or 0) / 60, "prime-sandbox"
+                    (self.config.creates_per_min or 0) / 60,
+                    "prime-sandbox",
+                    run_scope(),
                 )
                 or contextlib.nullcontext()
             ):

verifiers/v1/tasksets/__init__.py

@@ -1,10 +1,4 @@
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
-from verifiers.v1.tasksets.lean import (
-    LeanConfig,
-    LeanDatasetConfig,
-    LeanTask,
-    LeanTaskset,
-)
 from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset
 from verifiers.v1.tasksets.openenv import (
     OpenEnvConfig,
@@ -18,10 +12,6 @@ from verifiers.v1.tasksets.openenv import (
 __all__ = [
     "HarborConfig",
     "HarborTaskset",
-    "LeanConfig",
-    "LeanDatasetConfig",
-    "LeanTask",
-    "LeanTaskset",
     "NeMoGymConfig",
     "NeMoGymTaskset",
     "OpenEnvConfig",

verifiers/v1/utils/scope.py

@@ -0,0 +1,27 @@
+"""The scope that state shared by a run's processes is keyed by."""
+
+import logging
+import os
+import uuid
+
+logger = logging.getLogger(__name__)
+
+_process_scope: str | None = None
+
+
+def run_scope() -> str:
+    """``$VF_RUN_ID``, set by every entrypoint (the `vf-*` CLIs and the prime-rl launchers)
+    and inherited by spawned env servers and pool workers. A process started without one
+    gets a scope of its own, minted once per process."""
+    global _process_scope
+    run_id = os.environ.get("VF_RUN_ID")
+    if run_id:
+        return run_id
+    if _process_scope is None:
+        _process_scope = uuid.uuid4().hex
+        logger.warning(
+            "VF_RUN_ID is unset - run-scoped state such as creation limiters covers only "
+            "this process (scope %s)",
+            _process_scope,
+        )
+    return _process_scope

{verifiers-0.3.2.dev127.dist-info → verifiers-0.3.2.dev129.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev127
+Version: 0.3.2.dev129
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -48,8 +48,6 @@ Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvicorn>=0.52.0
 Provides-Extra: harbor
 Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
-Provides-Extra: lean
-Requires-Dist: datasets<6.0.0,>=3.3.0; extra == 'lean'
 Provides-Extra: modal
 Requires-Dist: modal>=1.5.4; extra == 'modal'
 Provides-Extra: openenv

{verifiers-0.3.2.dev127.dist-info → verifiers-0.3.2.dev129.dist-info}/RECORD

@@ -18,14 +18,14 @@ verifiers/v1/types.py,sha256=xPetvJxksKPYKic389RFnVvy2vN-gD2bPZFZoEyzmp4,10191
 verifiers/v1/acp/__init__.py,sha256=9RySmxFeEMT2XSJy5wYevqEhQdul2jHl9f5XAribG3A,13631
 verifiers/v1/acp/runner.py,sha256=zPo-2ZXmFMmQchhD7nNzlZUrabG09qiBvpB67KVGm3M,14095
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-verifiers/v1/cli/debug.py,sha256=3hPBoP8iceAUyx3lAjZ0z-QhiC5_z-jaRJfCTdxUFrk,11634
-verifiers/v1/cli/gepa.py,sha256=3-_GckCUttJRgKlsB9q_Ikhf4DtmTOIhyXOo6gNUiWc,4186
+verifiers/v1/cli/debug.py,sha256=sgXi2y17qr2yd5kbkL_IzwWN1nIFBOa1adu7bcFzaVk,11780
+verifiers/v1/cli/gepa.py,sha256=F0Q1ApfUL_mInFnmaJ5-yBDRYBtBuH72OkWelPoGCp4,4332
 verifiers/v1/cli/init.py,sha256=6wr14011_Dygv10R6_YWg5A3oSrz187E2WspGRHIfZw,7608
 verifiers/v1/cli/output.py,sha256=lAl6vwKf9mVuNZb8PzOexv6Q2h-H6t0oUPcapffY3uY,8351
-verifiers/v1/cli/replay.py,sha256=8EMROnzIWJ_2Mxs_eqZ19yProml9PZnxaIxHkqSwxXE,10249
+verifiers/v1/cli/replay.py,sha256=JjhaGczQ41zN6wz1P1nbGGvv0xdfQlX4OmZItGPDpZc,10395
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
 verifiers/v1/cli/resume.py,sha256=hqV5AxxqeIInJeEZQerQznuijhMNijFEFcHZIltfLdE,470
-verifiers/v1/cli/validate.py,sha256=K7mCz-ZI7sq69MQPg6QvreHOuEj-bgMln69a28kDTvA,17179
+verifiers/v1/cli/validate.py,sha256=Z_rUldl8jzXz52oy3FBIRGr4HYhWFXrq43WmU0sJ-4M,17325
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
 verifiers/v1/cli/dashboard/eval.py,sha256=HEXPEegNXdgU9zAIuHzsWsoSIHyiOcDZh4tCGFNxKAk,38353
@@ -35,7 +35,7 @@ verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 verifiers/v1/cli/eval/hint.py,sha256=NXP2_JYrQHguqr68LxOJG3lYAsue_VNoepyTVPn-N4E,256
 verifiers/v1/cli/eval/main.py,sha256=L_vKnMJEbpo1IKjbm0TXjXv_1x9we7mFv-jH_xY211Q,5946
 verifiers/v1/cli/eval/resume.py,sha256=fYkWa0Fudq--IWfk25j_McLm67bXYWvksWmJR4u4QAU,3811
-verifiers/v1/cli/eval/runner.py,sha256=gvwR3O6FM-yZMvS5uMgST3NGv5Egi7AG4cfYhP7bRC0,6831
+verifiers/v1/cli/eval/runner.py,sha256=z1yU2_rAA6tHz3PL7NQuzm9v3-4Ri18DXD5SRBKXqVY,6962
 verifiers/v1/clients/__init__.py,sha256=Ysig0tE_0E4Jsfgfes1XHN-fK1s_RCXdqZD6E7lK4PU,507
 verifiers/v1/clients/base.py,sha256=PoDw4GMqrfuPTFVK6n2jDmlD0_lJ5zTrYcNio5yq6K8,1691
 verifiers/v1/clients/client.py,sha256=zqC_AkiD9pl0kxxIbupNiehbS-YnVdp3EMZ0LOHTd8U,3125
@@ -79,7 +79,7 @@ verifiers/v1/envs/user_sim/env.py,sha256=deI4r-Utn_v4E_Vb9P_X3LzFh2GRkCO4OlQt8BO
 verifiers/v1/gepa/__init__.py,sha256=6nmdRE0-34AKioPHBjTxUg5Jo_2z7tMX-OU3zMNpAJI,197
 verifiers/v1/gepa/adapter.py,sha256=YNvHMR2L5Utl-vtPfjZxmDCd2tTcoAPWG6aVGnV1icM,6139
 verifiers/v1/gepa/config.py,sha256=PhZRLkhaPciNm_IhwVX3e9DM8Y2IdCe4Fbi5WjH5aro,4883
-verifiers/v1/gepa/dataset.py,sha256=hFuR7dou4VSKa0ujb8sFmGm9a19mjrsmTa1MMW1Kv7I,2177
+verifiers/v1/gepa/dataset.py,sha256=SPGG55BVbJ9HtCEvm-xyl52QSbldFwGqWvRIcaLelsM,2171
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
 verifiers/v1/gepa/runner.py,sha256=fGEyFzejdQKBzBbvdiTDHV3JtRhXMXMGSBU5y8C8i3w,6054
 verifiers/v1/harnesses/__init__.py,sha256=2JPwrRoYoigH-HfBtiFnatRdFEVHLv4rwlKfsFEibyE,1803
@@ -129,7 +129,7 @@ verifiers/v1/interception/server.py,sha256=wxw67OQkLgzvt1HMJyjYmdcoIhn4vuJBe5oXZ
 verifiers/v1/interception/tunnel/__init__.py,sha256=eVNZJszj6myrKhx9jmJz7VRd9qlfj0AUvUZwI2bXEj8,893
 verifiers/v1/interception/tunnel/base.py,sha256=DZB6uPLwM4Qy7n7m0vKeyg-Px87MhsmpYTpe2vpPEng,1998
 verifiers/v1/interception/tunnel/custom.py,sha256=yL4UbGf4bAuMsxk42mEP-t1wITQCT8BE9FHQIz0qRMA,1708
-verifiers/v1/interception/tunnel/prime.py,sha256=fIhU4Sa53f2FQWrwIgpDjh8XO9E9RdF9k1kh0TwDLTo,2721
+verifiers/v1/interception/tunnel/prime.py,sha256=jVk_6xyrxXiHtsVFErcRNd4MoENdFbqzfC0i20nlizU,2803
 verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
@@ -143,9 +143,9 @@ verifiers/v1/runtimes/__init__.py,sha256=JQlQ029J14asfYb-UxpENshMyPztWbSL0-ZqSyD
 verifiers/v1/runtimes/apptainer.py,sha256=Ru9QJna_l0goIzg71G-4RU_D-7wZUNz139S5TEvbzog,8211
 verifiers/v1/runtimes/base.py,sha256=IKmrJSJltWX9M1nPUZExKluPJM4N82VsnqQGFz0ECV8,17777
 verifiers/v1/runtimes/container.py,sha256=puQD8X9acP6GefoCNEnq40POi-nXPD-729bUjTVShfM,9554
-verifiers/v1/runtimes/limiters.py,sha256=C6cWDD4pwyFwUY4QFke3oiMXwydSNIfSbJSIglUC2kE,3062
-verifiers/v1/runtimes/modal.py,sha256=M1zBIMI6zJAq6io_t8uKGtXIxSZeEuAMuG3nR4L5pAU,17379
-verifiers/v1/runtimes/prime.py,sha256=BXwDK-eurEzOKLY1ezuC0UYZdEoI0Y9Cfcd0IhKjMPc,17586
+verifiers/v1/runtimes/limiters.py,sha256=rOfQJcMdYcLSKq8AL9O09BfGOLqpYmNgrCxXXNcHLmU,2849
+verifiers/v1/runtimes/modal.py,sha256=6ccitC0HVsAP_l8soJ_RQYNIVKrJjDsz2vAJBtxUZjw,17476
+verifiers/v1/runtimes/prime.py,sha256=nTtu-uBjc39UxfMPEyndshfAiIdk1V4L4aE6zSVVSn0,17673
 verifiers/v1/runtimes/subprocess.py,sha256=QmpQ235_xrX6gowrr8557VgbydOXpLbfFq0KKGfJThk,8774
 verifiers/v1/runtimes/docker/__init__.py,sha256=2wceP-8V-w0dQBEeLtoJFfCZ135DxP2rcdcUBmSWiNU,18543
 verifiers/v1/runtimes/docker/egress.py,sha256=wtKWTL2_zFidxMR7lDRmfevQlOsl2Iyk88ebdO4Kqqw,24566
@@ -156,14 +156,11 @@ verifiers/v1/serve/encoding.py,sha256=hBZFucAZK9riXOV3DaHcskq9zHTT8gVGkoFVOPfnr3
 verifiers/v1/serve/pool.py,sha256=bT-FOlIuOiBtOasuPB-p9887tkpEqvQUJBgn1Q45ph0,15912
 verifiers/v1/serve/server.py,sha256=i6XrY_PMAapkrP54tYX8-K6iGmOmrR7BbQ-3Qhn3w4s,9885
 verifiers/v1/serve/types.py,sha256=Qw5IjiJhsZz4giSXUBi9dfInBnQZ5_xiUmgH2qp8jPI,1741
-verifiers/v1/tasksets/__init__.py,sha256=5hTKSPLXSZTgKBbeKQI7FEp06c0ZsaQfuwvrvVFLgb8,712
+verifiers/v1/tasksets/__init__.py,sha256=2ijE3qIlLoUA02g6iysOwWQylyBBQL1GhSpjv8_bp0w,521
 verifiers/v1/tasksets/harbor/__init__.py,sha256=JxThzHFYEkuSfQbVf3pEnjordgdfe4MU-QOsSSbyovU,326
 verifiers/v1/tasksets/harbor/env.py,sha256=ATMkfwmrE_2jpOOffbjLQ91feG7TOtefBGlpwWbRq8c,3604
 verifiers/v1/tasksets/harbor/taskset.py,sha256=35ljhNapn1EOlaYFTuVZxAMVU0KDfmFxQOGOcUk2Ow0,34082
 verifiers/v1/tasksets/harbor/toolset.py,sha256=_7HXu-uuvqzhmJ1EX6c3KWLg0g0eRrLjCq1kBsZhIwo,3159
-verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
-verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
-verifiers/v1/tasksets/lean/taskset.py,sha256=Jax76i22G9dOAswQHiWujVbFIM_PUz5yGJI_ShR4n8Y,9238
 verifiers/v1/tasksets/nemo_gym/__init__.py,sha256=654gAvG3Wd_Z_nh6mypeKn4s_IIQ0Ge6vcZvxTHC2_E,306
 verifiers/v1/tasksets/nemo_gym/response.py,sha256=EhrsquEP8g0bjzAThGfE6Ypgy86_nVX235NiFYtVDVM,4463
 verifiers/v1/tasksets/nemo_gym/server.py,sha256=33C-VDZxPtMV2TFxJVIKijmtBLNlVeRs92ctPE3qAsM,2036
@@ -190,10 +187,11 @@ verifiers/v1/utils/paths.py,sha256=it_4JWf_8bPZe4TRcyvL7_KJ4fsbEsxhhIY_xA8Ikus,3
 verifiers/v1/utils/platform.py,sha256=YwTHVA1Onn74lG6S3sdHus9OlP-rAbwkk9Po7h-BjpE,7159
 verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,970
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
+verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,838
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev127.dist-info/METADATA,sha256=F8kPNYPLJWPcdQ0mwZw2mQ7hKpVaOQ9ap_ASg6SmLyo,4237
-verifiers-0.3.2.dev127.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev127.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.3.2.dev127.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev127.dist-info/RECORD,,
+verifiers-0.3.2.dev129.dist-info/METADATA,sha256=ckfiJIStyeQf1e_fIyVWL-7jtc5yiEqtyEWdvxHCNKQ,4161
+verifiers-0.3.2.dev129.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev129.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
+verifiers-0.3.2.dev129.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev129.dist-info/RECORD,,

verifiers/v1/tasksets/lean/__init__.py

@@ -1,33 +0,0 @@
-from verifiers.v1.tasksets.lean.scoring import (
-    build_starter_file,
-    expected_protected_signature,
-    parse_compile_output,
-    protected_signature_substring_present,
-    strip_lean_comments,
-)
-from verifiers.v1.tasksets.lean.taskset import (
-    DEFAULT_DOCKER_IMAGE,
-    LEAN_PROJECT_PATH,
-    PROOF_FILE_PATH,
-    LeanConfig,
-    LeanDatasetConfig,
-    LeanTask,
-    LeanTaskConfig,
-    LeanTaskset,
-)
-
-__all__ = [
-    "DEFAULT_DOCKER_IMAGE",
-    "LEAN_PROJECT_PATH",
-    "PROOF_FILE_PATH",
-    "LeanConfig",
-    "LeanDatasetConfig",
-    "LeanTask",
-    "LeanTaskConfig",
-    "LeanTaskset",
-    "build_starter_file",
-    "expected_protected_signature",
-    "parse_compile_output",
-    "protected_signature_substring_present",
-    "strip_lean_comments",
-]

verifiers/v1/tasksets/lean/scoring.py

@@ -1,262 +0,0 @@
-"""Pure helpers for the Lean taskset: starter-file construction, theorem-signature
-canonicalization, the reward-hacking signature guard, and compile-output parsing.
-
-Everything here is a pure function over plain strings — no runtime, no sandbox, no
-verifiers types — so it's trivially unit-testable and shared between ``setup``
-(building the starter file), the reward (the signature guard + compile parsing),
-and ``validate`` (planting the gold proof).
-"""
-
-from __future__ import annotations
-
-import re
-
-PROTECTED_HEADER_COMMENT = (
-    "-- DO NOT MODIFY the theorem statement below. The grader checks\n"
-    "-- that the original `theorem ... := by` text still appears in\n"
-    "-- this file. Only edit the proof body (currently `sorry`) and\n"
-    "-- lines after it."
-)
-
-
-# ── Imports / preamble normalization ─────────────────────────────────────────
-
-
-def normalize_imports(text: str) -> str:
-    """Collapse every ``import Mathlib*`` line to a single ``import Mathlib``.
-
-    Some datasets (miniF2F) ship fine-grained Mathlib imports that don't resolve
-    against the monolithic ``import Mathlib`` the sandbox image provides.
-    """
-    lines = text.split("\n")
-    out: list[str] = []
-    inserted = False
-    for line in lines:
-        if line.strip().startswith("import Mathlib"):
-            if not inserted:
-                out.append("import Mathlib")
-                inserted = True
-        else:
-            out.append(line)
-    return "\n".join(out)
-
-
-def _build_preamble(imports_str: str, header: str, normalize: bool) -> str:
-    if header and header.strip().startswith("import"):
-        preamble = header.strip()
-        return normalize_imports(preamble) if normalize else preamble
-    parts = [imports_str.strip()]
-    if header and header.strip():
-        parts.append(header.strip())
-    preamble = "\n\n".join(parts)
-    return normalize_imports(preamble) if normalize else preamble
-
-
-# ── Theorem signature canonicalization ───────────────────────────────────────
-
-
-def _normalize_signature(stmt: str) -> str:
-    """Canonicalize a Lean theorem statement to end with ``:= by``.
-
-    Strips trailing ``sorry``/``admit`` placeholders and any trailing ``by`` /
-    ``:=`` tokens, then re-appends `` := by``. Places the appended token on a new
-    indented line when the last line of the stripped signature already contains a
-    ``--`` comment (otherwise the ``:= by`` would land inside the line comment and
-    Lean would silently ignore it).
-    """
-    s = stmt.rstrip()
-    s = re.sub(r"\s*\b(?:sorry|admit)\b\s*$", "", s)
-    s = re.sub(r"\s*\bby\b\s*$", "", s)
-    s = re.sub(r"\s*:=\s*$", "", s)
-    s = s.rstrip()
-    last_newline = s.rfind("\n")
-    last_line = s[last_newline + 1 :] if last_newline != -1 else s
-    sep = "\n    " if "--" in last_line else " "
-    return s + sep + ":= by"
-
-
-def _split_imports_and_signature(stmt: str) -> tuple[str, str]:
-    decl_match = re.search(r"^(?:theorem|lemma|example)\s", stmt, flags=re.MULTILINE)
-    if not decl_match:
-        return "", stmt
-    return stmt[: decl_match.start()].rstrip(), stmt[decl_match.start() :]
-
-
-def expected_protected_signature(formal_statement: str) -> str:
-    """Return the canonical ``theorem ... := by`` block the reward pins as ground truth.
-
-    Pure function over ``formal_statement``; the imports/header portion (if the
-    statement carries one inline) is stripped first.
-    """
-    stmt = formal_statement or ""
-    if stmt.strip().startswith("import "):
-        _, signature_raw = _split_imports_and_signature(stmt)
-    else:
-        signature_raw = stmt
-    return _normalize_signature(signature_raw).strip()
-
-
-# ── Comment / string stripping for the signature guard ───────────────────────
-
-
-def strip_lean_comments(text: str) -> str:
-    """Remove Lean line/block comments **and string literals** from ``text``.
-
-    Lean comments come in two forms: ``-- ...`` line comments (to end of line) and
-    ``/- ... -/`` block comments (nestable; ``/-- ... -/`` doc comments are a
-    special case). String literals must also be stripped: a Lean
-    ``"theorem ... := by"`` constant or doc-string would otherwise let a model hide
-    the pinned signature inside a string while rewriting the live declaration to a
-    trivial one, defeating the substring guard. We handle both regular
-    double-quoted strings (with backslash escapes) and triple-quoted raw strings.
-
-    Block comments nest, so we count depth; line comments end at the next newline.
-    Outside comments and strings, newlines are preserved so the result keeps
-    roughly the right shape for substring matching downstream.
-    """
-    out: list[str] = []
-    i = 0
-    n = len(text)
-    block_depth = 0
-    in_line_comment = False
-    while i < n:
-        ch = text[i]
-        if in_line_comment:
-            if ch == "\n":
-                in_line_comment = False
-                out.append(ch)
-            i += 1
-            continue
-        if block_depth > 0:
-            if i + 1 < n and text[i : i + 2] == "-/":
-                block_depth -= 1
-                i += 2
-                continue
-            if i + 1 < n and text[i : i + 2] == "/-":
-                block_depth += 1
-                i += 2
-                continue
-            if ch == "\n":
-                out.append(ch)
-            i += 1
-            continue
-        if i + 1 < n and text[i : i + 2] == "/-":
-            block_depth = 1
-            i += 2
-            continue
-        if i + 1 < n and text[i : i + 2] == "--":
-            in_line_comment = True
-            i += 2
-            continue
-        # Triple-quoted raw string ``"""..."""`` — skip until the closing
-        # triple-quote, preserving newlines.
-        if i + 2 < n and text[i : i + 3] == '"""':
-            i += 3
-            while i < n:
-                if i + 2 < n and text[i : i + 3] == '"""':
-                    i += 3
-                    break
-                if text[i] == "\n":
-                    out.append("\n")
-                i += 1
-            continue
-        # Regular string ``"..."`` — handle ``\\`` escapes, stop at the next
-        # unescaped quote or newline (Lean strings are single-line).
-        if ch == '"':
-            i += 1
-            while i < n and text[i] != '"' and text[i] != "\n":
-                if text[i] == "\\" and i + 1 < n:
-                    i += 2
-                else:
-                    i += 1
-            if i < n and text[i] == '"':
-                i += 1
-            continue
-        out.append(ch)
-        i += 1
-    return "".join(out)
-
-
-def protected_signature_substring_present(
-    content: str, expected_signature: str
-) -> bool:
-    """True when the locked signature text still appears in the file.
-
-    Strips Lean comments from BOTH sides — without that a model could paste the
-    pinned signature into a ``--`` or ``/- ... -/`` block while rewriting the live
-    declaration to something trivial, and the asymmetric variant would also misfire
-    if ``expected_signature`` itself contains a comment. Tries an exact substring
-    match first, then a whitespace-flexible match (each side collapsed to
-    single-spaced tokens) so the model can re-indent or reflow whitespace freely.
-    """
-    if not expected_signature:
-        return True
-    decommented_content = strip_lean_comments(content)
-    decommented_expected = strip_lean_comments(expected_signature)
-    if not decommented_expected.strip():
-        return True
-    if decommented_expected in decommented_content:
-        return True
-    flat_signature = " ".join(decommented_expected.split())
-    flat_content = " ".join(decommented_content.split())
-    return flat_signature in flat_content
-
-
-# ── Starter-file construction ────────────────────────────────────────────────
-
-
-def build_starter_file(
-    formal_statement: str,
-    header: str = "",
-    imports: str = "import Mathlib",
-    normalize: bool = False,
-    proof_body: str | None = None,
-) -> str:
-    """Construct the starter proof file.
-
-    Layout: preamble (imports / header) + a brief ``-- DO NOT MODIFY`` comment
-    block + the normalized theorem signature + the proof body. If ``proof_body`` is
-    None (the default) the body is the placeholder ``  sorry`` — what's planted at
-    rollout start. A supplied gold ``proof_body`` (e.g. by ``validate``) replaces
-    the placeholder so the file is the full reference solution.
-    """
-    stmt = formal_statement or ""
-    if stmt.strip().startswith("import "):
-        imports_block, signature_raw = _split_imports_and_signature(stmt)
-        preamble = normalize_imports(imports_block) if normalize else imports_block
-    else:
-        preamble = _build_preamble(imports or "import Mathlib", header, normalize)
-        signature_raw = stmt
-
-    signature = _normalize_signature(signature_raw)
-    body = "  sorry" if proof_body is None else proof_body.rstrip()
-    wrapped = f"{PROTECTED_HEADER_COMMENT}\n{signature}\n{body}\n"
-    if preamble:
-        return preamble.rstrip() + "\n\n" + wrapped
-    return wrapped
-
-
-# ── Compile-output parsing ───────────────────────────────────────────────────
-
-
-def parse_compile_output(output: str) -> tuple[bool, str, int]:
-    """Parse a ``lake env lean ...; echo EXIT_CODE:$?`` transcript.
-
-    Returns ``(compiled, cleaned_output, exit_code)`` where ``compiled`` is True iff
-    the compiler exited 0 with no ``declaration uses 'sorry'`` diagnostic.
-
-    Matches the LAST ``EXIT_CODE:N`` — that's the one our shell appends at the end
-    of the command. Matching the first occurrence would let a model inject
-    ``#eval IO.println "EXIT_CODE:0"`` into the proof file to bypass the
-    sorry/exit-code checks: the regex would hit the injected marker, truncate
-    everything after it (hiding the real ``declaration uses 'sorry'`` diagnostic and
-    the real EXIT_CODE), and report success.
-    """
-    exit_code = 1
-    matches = list(re.finditer(r"EXIT_CODE:(\d+)", output))
-    if matches:
-        last = matches[-1]
-        exit_code = int(last.group(1))
-        output = output[: last.start()].strip()
-    has_sorry = bool(re.search(r"declaration uses 'sorry'", output))
-    return (exit_code == 0 and not has_sorry), output, exit_code

verifiers/v1/tasksets/lean/taskset.py

@@ -1,229 +0,0 @@
-"""Lean 4 theorem-proving tasks backed by Hugging Face datasets.
-
-Each task plants a ``sorry``-based starter file, lets any container-capable
-harness edit it, and rewards a clean ``lake env lean`` compile. The original
-theorem signature must remain present, which prevents replacing the assigned
-statement with an easier theorem. Dataset-specific packages can subclass this
-taskset and only supply column mappings.
-"""
-
-from __future__ import annotations
-
-import shlex
-from collections.abc import Iterator
-
-from pydantic_config import BaseConfig
-
-from verifiers.v1.configs.task import TaskConfig
-from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.runtimes import Runtime
-from verifiers.v1.state import State
-from verifiers.v1.task import Task, TaskData, TaskResources
-from verifiers.v1.taskset import Taskset
-from verifiers.v1.tasksets.lean.scoring import (
-    build_starter_file,
-    expected_protected_signature,
-    parse_compile_output,
-    protected_signature_substring_present,
-)
-from verifiers.v1.trace import Trace
-from verifiers.v1.utils.decorators import reward
-
-# Lean v4.27 with Mathlib v4.27.
-DEFAULT_DOCKER_IMAGE = "team-clyvldofb0000gg1kx39rgzjq/lean-tactic:mathlib-v4.27.0-v3"
-LEAN_PROJECT_PATH = "/workspace/mathlib4"
-PROOF_FILE_PATH = "/tmp/proof.lean"
-
-DEFAULT_SYSTEM_PROMPT = "You are an expert Lean 4 theorem prover working with Mathlib."
-
-
-class LeanDatasetConfig(BaseConfig):
-    name: str
-    """HuggingFace dataset id (required; each per-dataset package sets it)."""
-    split: str = "train"
-    subset: str | None = None
-    statement_column: str = "formal_statement"
-    header_column: str | None = None
-    imports_column: str | None = None
-    name_column: str | None = None
-    proof_column: str | None = None
-    """Column holding the gold proof body (used by ``validate``); None = no gold."""
-    normalize_mathlib_imports: bool = False
-
-
-class LeanTaskConfig(TaskConfig):
-    lean_project_path: str = LEAN_PROJECT_PATH
-    proof_file_path: str = PROOF_FILE_PATH
-    compile_timeout: int = 300
-    """Per-compile ``timeout`` wrapper (seconds), bounding each ``lake env lean``."""
-
-
-class LeanConfig(TasksetConfig):
-    dataset: LeanDatasetConfig
-    docker_image: str = DEFAULT_DOCKER_IMAGE
-    task: LeanTaskConfig = LeanTaskConfig()
-
-
-class LeanData(TaskData):
-    formal_statement: str
-    header: str = ""
-    imports: str = "import Mathlib"
-    normalize_mathlib_imports: bool = False
-    # Canonical ``theorem ... := by`` text pinned at load; the reward checks it
-    # still appears in the final file (the only edit the reward cares about).
-    protected_signature: str = ""
-    # Gold proof body (replaces ``  sorry``); "" when the dataset ships no gold.
-    formal_proof: str = ""
-
-
-class LeanTask(Task[LeanData, State, LeanTaskConfig]):
-    NEEDS_CONTAINER = True
-
-    async def _compile(self, runtime: Runtime) -> tuple[bool, str, int]:
-        cmd = (
-            f"cd {shlex.quote(self.config.lean_project_path)} && "
-            f"timeout {self.config.compile_timeout} lake env lean "
-            f"{shlex.quote(self.config.proof_file_path)} 2>&1; "
-            "echo EXIT_CODE:$?"
-        )
-        result = await runtime.run(["bash", "-lc", cmd], {})
-        return parse_compile_output((result.stdout or "") + (result.stderr or ""))
-
-    async def setup(self, runtime: Runtime) -> None:
-        content = build_starter_file(
-            self.data.formal_statement,
-            header=self.data.header,
-            imports=self.data.imports,
-            normalize=self.data.normalize_mathlib_imports,
-        )
-        await runtime.write(self.config.proof_file_path, content.encode())
-
-    @reward(weight=1.0)
-    async def lean_compiled(self, trace: Trace, runtime: Runtime) -> float:
-        """Require both the assigned signature and a clean Lean compile."""
-        if trace.has_error:
-            return 0.0
-
-        # Setup created this file, so a read failure is an infrastructure error.
-        current = (await runtime.read(self.config.proof_file_path)).decode(
-            "utf-8", "replace"
-        )
-
-        expected_sig = self.data.protected_signature or expected_protected_signature(
-            self.data.formal_statement
-        )
-        if expected_sig and not protected_signature_substring_present(
-            current, expected_sig
-        ):
-            trace.info["lean_tampered"] = True
-            trace.info["compile_output"] = "signature rewritten or hidden in a comment"
-            return 0.0
-        trace.info["lean_tampered"] = False
-
-        compiled, output, exit_code = await self._compile(runtime)
-        trace.info["lean_compiled"] = compiled
-        trace.info["compile_exit_code"] = exit_code
-        trace.info["compile_output"] = output[-4000:]
-        return 1.0 if compiled else 0.0
-
-    async def validate(self, runtime: Runtime) -> bool | None:
-        """Compile the gold proof; rows without one have nothing to preflight."""
-        gold = (self.data.formal_proof or "").rstrip()
-        if not gold:
-            return None
-        content = build_starter_file(
-            self.data.formal_statement,
-            header=self.data.header,
-            imports=self.data.imports,
-            normalize=self.data.normalize_mathlib_imports,
-            proof_body=gold,
-        )
-        await runtime.write(self.config.proof_file_path, content.encode())
-        compiled, _, _ = await self._compile(runtime)
-        return compiled
-
-
-class LeanTaskset(Taskset[LeanTask, LeanConfig]):
-    def load(self) -> Iterator[LeanTask]:
-        try:
-            from datasets import load_dataset
-        except ModuleNotFoundError as e:
-            raise ModuleNotFoundError(
-                "the Lean taskset requires the `lean` extra; install `verifiers[lean]`"
-            ) from e
-
-        config = self.config
-        ds = config.dataset
-        raw = load_dataset(
-            ds.name,
-            ds.subset,
-            split=ds.split,
-            num_proc=8,
-        )
-
-        # Validate optional columns before empty-value fallbacks hide a typo.
-        for label, col in (
-            ("statement_column", ds.statement_column),
-            ("header_column", ds.header_column),
-            ("imports_column", ds.imports_column),
-            ("name_column", ds.name_column),
-            ("proof_column", ds.proof_column),
-        ):
-            if col is not None and col not in raw.column_names:
-                raise ValueError(
-                    f"dataset.{label}={col!r} not found in {ds.name!r}; columns={raw.column_names}"
-                )
-
-        resources = TaskResources(cpu=4, memory=4, disk=10)
-        for index, row in enumerate(raw):
-            # An empty statement would reduce the signature guard to `:= by`.
-            formal_statement = row[ds.statement_column]
-            if not isinstance(formal_statement, str) or not formal_statement.strip():
-                continue
-            # A disabled optional column is None; row.get(None) yields the same
-            # empty fallback as a present-but-empty column.
-            header = row.get(ds.header_column) or ""
-            imports = row.get(ds.imports_column) or "import Mathlib"
-            gold = row.get(ds.proof_column) or ""
-            name = row.get(ds.name_column)
-            yield LeanTask(
-                LeanData(
-                    idx=index,
-                    name=str(name) if name else f"task_{index:05d}",
-                    prompt=self._build_prompt(formal_statement, header),
-                    system_prompt=DEFAULT_SYSTEM_PROMPT,
-                    image=config.docker_image,
-                    workdir=config.task.lean_project_path,
-                    resources=resources,
-                    formal_statement=formal_statement,
-                    header=header,
-                    imports=imports,
-                    normalize_mathlib_imports=ds.normalize_mathlib_imports,
-                    protected_signature=expected_protected_signature(formal_statement),
-                    formal_proof=gold,
-                ),
-                self.config.task,
-            )
-
-    def _build_prompt(self, formal_statement: str, header: str) -> str:
-        cfg = self.config
-        block = (
-            "Prove the following Lean 4 theorem. A starter proof file is at "
-            f"`{cfg.task.proof_file_path}` with the theorem statement and a `sorry` "
-            "placeholder already in place. Edit it and compile with "
-            f"`cd {cfg.task.lean_project_path} && lake env lean {cfg.task.proof_file_path}`.\n\n"
-            f"```lean\n{formal_statement}\n```"
-        )
-        if header:
-            block += f"\n\nThe file header (imports/namespaces) is already set up:\n```lean\n{header}\n```"
-        block += (
-            "\n\nDo NOT modify the theorem statement (the lines from `theorem ...` "
-            "through `:= by`) — the grader checks the original statement still "
-            "appears and gives zero reward if you rewrote it. Write your proof "
-            "tactics in place of `sorry`; the final proof must not contain `sorry` "
-            "or `admit`. A clean compile prints nothing and exits 0."
-        )
-        return block
-
-
-__all__ = ["LeanConfig", "LeanTask", "LeanTaskset"]