verifiers 0.3.2.dev127__tar.gz → 0.3.2.dev129__tar.gz

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev127
+Version: 0.3.2.dev129
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -48,8 +48,6 @@ Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvicorn>=0.52.0
 Provides-Extra: harbor
 Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
-Provides-Extra: lean
-Requires-Dist: datasets<6.0.0,>=3.3.0; extra == 'lean'
 Provides-Extra: modal
 Requires-Dist: modal>=1.5.4; extra == 'modal'
 Provides-Extra: openenv

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/pyproject.toml

@@ -89,9 +89,6 @@ examples = [
 ]
 
 [project.optional-dependencies]
-lean = [
-    "datasets>=3.3.0,<6.0.0",
-]
 ta = [
     "textarena==0.7.4",
     "nltk>=3.9.2",

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/cli/debug.py

@@ -3,10 +3,12 @@
 import asyncio
 import contextlib
 import logging
+import os
 import shlex
 import sys
 import time
 import traceback
+import uuid
 from collections.abc import Awaitable
 from pathlib import Path
 from typing import Any
@@ -316,6 +318,8 @@ async def run_debug(config: DebugConfig) -> list[Trace]:
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = with_positional_taskset(
         list(sys.argv[1:]) if argv is None else list(argv), flag="--taskset.id"
     )

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/cli/eval/runner.py

@@ -9,6 +9,7 @@ runs env servers); this CLI is the quick local path.
 import asyncio
 import contextlib
 import logging
+import os
 import time
 from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
 from typing import TypeVar, cast
@@ -139,6 +140,8 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
 
     # Opened before the first rollout so every episode streams as it lands.
     run = open_run(config, push_state, num_examples=len(tasks))
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", config.run.id)
     # Resumed rollouts are part of this run too.
     log_episodes(run, finished)
 

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/cli/gepa.py

@@ -9,7 +9,9 @@ and the actual parse is `pydantic_config.cli`.
 """
 
 import logging
+import os
 import sys
+import uuid
 
 from pydantic_config import cli
 
@@ -32,6 +34,8 @@ USAGE = "usage: uv run vf-gepa [<taskset-id>] [--env.id <id>] --model <model> [o
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
 
     if not argv or any(arg in ("-h", "--help") for arg in argv):

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/cli/replay.py

@@ -12,8 +12,10 @@ import asyncio
 import contextlib
 import json
 import logging
+import os
 import sys
 import time
+import uuid
 from pathlib import Path
 
 from pydantic_config import cli
@@ -197,6 +199,8 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = list(sys.argv[1:]) if argv is None else list(argv)
     if not argv or any(a in ("-h", "--help") for a in argv):
         print(USAGE)

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/cli/validate.py

@@ -4,9 +4,11 @@ import asyncio
 import contextlib
 import json
 import logging
+import os
 import shutil
 import sys
 import time
+import uuid
 from collections import Counter, defaultdict
 from collections.abc import Mapping, Sequence
 from pathlib import Path
@@ -425,6 +427,8 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
 
 
 def main(argv: list[str] | None = None) -> None:
+    # The run identity: every process this run spawns inherits it.
+    os.environ.setdefault("VF_RUN_ID", uuid.uuid4().hex)
     argv = with_positional_taskset(
         list(sys.argv[1:]) if argv is None else list(argv), flag="--taskset.id"
     )

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/gepa/dataset.py

@@ -40,5 +40,5 @@ def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> s
         "no task in this taskset sets Task.system_prompt — some tasksets bake instructions "
         "directly into `prompt` instead (e.g. gsm8k) and can't be optimized this way. Pass "
         "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets "
-        "system_prompt on its task data (e.g. reverse-text, lean, textarena)."
+        "system_prompt on its task data (e.g. reverse-text, textarena)."
     )

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/interception/tunnel/prime.py

@@ -9,15 +9,19 @@ from collections.abc import AsyncIterator
 from typing import Literal
 
 from verifiers.v1.interception.tunnel.base import BaseTunnelConfig, Tunnel
-from verifiers.v1.runtimes.limiters import creation_limiter
+from verifiers.v1.runtimes.limiters import CreationLimiter
 from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.prime import ensure_prime_auth
+from verifiers.v1.utils.scope import run_scope
 
 # The prime_tunnel service caps tunnel starts at 512/min per API token — a property of the
-# tunnel service, shared by every process for the user that opens one. One user-global
+# tunnel service, shared by every process of a run that opens one. One run-scoped
 # limiter, not a per-runtime config knob.
 _TUNNELS_PER_MIN = 512
-TUNNEL_LIMITER = creation_limiter(_TUNNELS_PER_MIN / 60, "prime-tunnel")
+
+
+def tunnel_limiter() -> CreationLimiter:
+    return CreationLimiter("prime-tunnel", run_scope(), _TUNNELS_PER_MIN / 60)
 
 
 class PrimeTunnelConfig(BaseTunnelConfig):
@@ -35,8 +39,8 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
     @contextlib.asynccontextmanager
     async def expose(self, port: int) -> AsyncIterator[str]:
         """Bridge the host `port` to a public URL via prime_tunnel (frpc). Tunnel creation
-        is network-bound and globally rate-capped (512/min, user-wide via the shared
-        `TUNNEL_LIMITER`), so transient failures are retried; a terminal one raises
+        is network-bound and rate-capped (512/min, run-wide via the shared
+        `tunnel_limiter`), so transient failures are retried; a terminal one raises
         `TunnelError`. The tunnel is torn down on exit."""
         from prime_tunnel import Tunnel as TunnelClient
 
@@ -48,7 +52,7 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
             async for attempt in retrying(retries=3, label=label):
                 with attempt:
                     client = TunnelClient(local_port=port)
-                    async with TUNNEL_LIMITER:
+                    async with tunnel_limiter():
                         url = str(await client.start()).rstrip("/")
         except Exception as e:
             raise TunnelError(f"{label} failed: {e}") from e

verifiers-0.3.2.dev129/verifiers/v1/runtimes/limiters.py

@@ -0,0 +1,71 @@
+"""Creation-rate limiters for the remote runtimes.
+
+A leaky bucket backed by a lock file under the user cache (``~/.cache/verifiers``, falling
+back to the temp dir when no home is resolvable), so a provider's creation rate (Modal
+sandboxes, Prime sandboxes and tunnels) is enforced across EVERY process that shares the
+bucket — the eval process and all the elastically-spawned env-server worker processes alike —
+not just within one process. A bucket is named by the limiter's name and a scope the caller
+picks; the runtimes pass the run id, so one run's backlog (or the reservations a killed run
+left behind) never delays another run.
+"""
+
+import asyncio
+import fcntl
+import os
+import time
+from typing import Self
+
+from verifiers.v1.utils.paths import CACHE_DIR
+
+LIMITER_DIR = CACHE_DIR / "limiter"
+
+
+class CreationLimiter:
+    """An async leaky bucket shared across processes via a lock file: each `async with`
+    reserves the next `1/per_sec`-spaced slot (advancing the on-disk cursor under an exclusive
+    flock) and sleeps until it, so the aggregate creation rate across every process sharing
+    the bucket stays at `per_sec`. The reservation runs off the event loop; the wait does not
+    hold the lock. Reservations are never released, so a cancelled waiter still holds its
+    slot; the backlog drains at `per_sec` regardless."""
+
+    def __init__(self, name: str, scope: str, per_sec: float) -> None:
+        self._interval = 1 / per_sec
+        self._path = LIMITER_DIR / f"{name}-{scope.replace('/', '--')}.bucket"
+
+    def _reserve(self) -> float:
+        os.makedirs(LIMITER_DIR, exist_ok=True)
+        # Shared buckets require a clock comparable across the run's hosts.
+        with open(self._path, "a+") as f:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+            try:
+                f.seek(0)
+                data = f.read().strip()
+                now = time.time()
+                slot = max(now, float(data) if data else 0.0)
+                f.seek(0)
+                f.truncate()
+                f.write(repr(slot + self._interval))
+                f.flush()
+            finally:
+                fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+        return slot - now
+
+    async def __aenter__(self) -> Self:
+        wait = await asyncio.to_thread(self._reserve)
+        if wait > 0:
+            await asyncio.sleep(wait)
+        return self
+
+    async def __aexit__(self, *exc) -> bool:
+        return False
+
+
+def creation_limiter(
+    per_sec: float | None, name: str, scope: str
+) -> CreationLimiter | None:
+    """A limiter pacing `name`'s creation to `per_sec`/s within `scope` (None/<= 0
+    disables). All callers (and processes) sharing a name and scope share one bucket, so
+    use one rate per name."""
+    if not per_sec or per_sec <= 0:
+        return None
+    return CreationLimiter(name, scope, per_sec)

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/runtimes/modal.py

@@ -31,6 +31,7 @@ from verifiers.v1.runtimes.base import (
 )
 from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.scope import run_scope
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +94,7 @@ class ModalConfig(NetworkPolicyConfig):
     """Disk in GB. Modal sandboxes have no disk knob, so this is accepted (so a task can
     declare it without a warning) but not enforced."""
     creates_per_sec: float | None = 40.0
-    """Pace sandbox creation to this many per second, enforced user-wide across every
+    """Pace sandbox creation to this many per second, enforced run-wide across every
     env-server worker process (None/<= 0 disables it)."""
 
     @model_validator(mode="after")
@@ -181,7 +182,9 @@ class ModalRuntime(Runtime):
         try:
             app = await modal.App.lookup.aio(_APP_NAME, create_if_missing=True)
             async with (
-                creation_limiter(self.config.creates_per_sec, "modal-sandbox")
+                creation_limiter(
+                    self.config.creates_per_sec, "modal-sandbox", run_scope()
+                )
                 or contextlib.nullcontext()
             ):
                 await run_shielded(self._create_sandbox(app))

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/runtimes/prime.py

@@ -28,6 +28,7 @@ from verifiers.v1.runtimes.base import (
 from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.prime import ensure_prime_auth
+from verifiers.v1.utils.scope import run_scope
 
 logger = logging.getLogger(__name__)
 
@@ -87,9 +88,9 @@ class PrimeConfig(NetworkPolicyConfig):
     idle_timeout: float | None = 3600
     """Seconds of inactivity before the sandbox self-deletes (None disables)."""
     creates_per_min: int | None = None
-    """Pace sandbox creation to this many per minute, enforced user-wide across every
+    """Pace sandbox creation to this many per minute, enforced run-wide across every
     env-server worker process (None/<= 0 disables it). (Tunnel creation is limited separately
-    and globally — see interception.tunnel.prime.TUNNEL_LIMITER.)"""
+    — see interception.tunnel.prime.tunnel_limiter.)"""
 
     @model_validator(mode="after")
     def _validate_egress(self) -> "PrimeConfig":
@@ -175,7 +176,9 @@ class PrimeRuntime(Runtime):
         try:
             async with (
                 creation_limiter(
-                    (self.config.creates_per_min or 0) / 60, "prime-sandbox"
+                    (self.config.creates_per_min or 0) / 60,
+                    "prime-sandbox",
+                    run_scope(),
                 )
                 or contextlib.nullcontext()
             ):

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev129}/verifiers/v1/tasksets/__init__.py

@@ -1,10 +1,4 @@
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
-from verifiers.v1.tasksets.lean import (
-    LeanConfig,
-    LeanDatasetConfig,
-    LeanTask,
-    LeanTaskset,
-)
 from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset
 from verifiers.v1.tasksets.openenv import (
     OpenEnvConfig,
@@ -18,10 +12,6 @@ from verifiers.v1.tasksets.openenv import (
 __all__ = [
     "HarborConfig",
     "HarborTaskset",
-    "LeanConfig",
-    "LeanDatasetConfig",
-    "LeanTask",
-    "LeanTaskset",
     "NeMoGymConfig",
     "NeMoGymTaskset",
     "OpenEnvConfig",

verifiers-0.3.2.dev129/verifiers/v1/utils/scope.py

@@ -0,0 +1,27 @@
+"""The scope that state shared by a run's processes is keyed by."""
+
+import logging
+import os
+import uuid
+
+logger = logging.getLogger(__name__)
+
+_process_scope: str | None = None
+
+
+def run_scope() -> str:
+    """``$VF_RUN_ID``, set by every entrypoint (the `vf-*` CLIs and the prime-rl launchers)
+    and inherited by spawned env servers and pool workers. A process started without one
+    gets a scope of its own, minted once per process."""
+    global _process_scope
+    run_id = os.environ.get("VF_RUN_ID")
+    if run_id:
+        return run_id
+    if _process_scope is None:
+        _process_scope = uuid.uuid4().hex
+        logger.warning(
+            "VF_RUN_ID is unset - run-scoped state such as creation limiters covers only "
+            "this process (scope %s)",
+            _process_scope,
+        )
+    return _process_scope

verifiers-0.3.2.dev127/verifiers/v1/runtimes/limiters.py

@@ -1,79 +0,0 @@
-"""User-global creation-rate limiters for the remote runtimes.
-
-A leaky bucket backed by a lock file under the user cache (``~/.cache/verifiers``, falling
-back to the temp dir when no home is resolvable), so a provider's per-account creation rate
-(Modal sandboxes, Prime tunnels) is enforced across EVERY process for the user —
-the single-process eval and all the elastically-spawned env-server worker processes alike — not
-just within one process. Keyed by name: one bucket file per name, shared by every process (and
-run) for the user.
-"""
-
-import asyncio
-import fcntl
-import os
-import time
-from typing import Self
-
-from verifiers.v1.utils.paths import CACHE_DIR
-
-LIMITER_DIR = CACHE_DIR / "limiter"
-
-
-class CreationLimiter:
-    """An async leaky bucket shared across processes via a lock file: each `async with`
-    reserves the next `1/per_sec`-spaced slot (advancing the on-disk cursor under an exclusive
-    flock) and sleeps until it, so the aggregate creation rate across all of the user's
-    processes stays at `per_sec`. The reservation runs off the event loop; the wait does not
-    hold the lock. Backlogs over five minutes fail rather than silently stalling creation."""
-
-    def __init__(self, name: str, per_sec: float) -> None:
-        self._interval = 1 / per_sec
-        self._path = LIMITER_DIR / f"{name}.bucket"
-
-    def _reserve(self) -> float:
-        os.makedirs(LIMITER_DIR, exist_ok=True)
-        # Shared buckets require a clock comparable across hosts and boots.
-        with open(self._path, "a+") as f:
-            fcntl.flock(f.fileno(), fcntl.LOCK_EX)
-            try:
-                f.seek(0)
-                data = f.read().strip()
-                now = time.time()
-                slot = max(now, float(data) if data else 0.0)
-                wait = slot - now
-                if wait > 5 * 60:
-                    raise TimeoutError(
-                        f"{self._path.stem} creation limiter backlog of {wait:.1f}s "
-                        f"exceeds 300s ({self._path})"
-                    )
-                f.seek(0)
-                f.truncate()
-                f.write(repr(slot + self._interval))
-                f.flush()
-                return wait
-            finally:
-                fcntl.flock(f.fileno(), fcntl.LOCK_UN)
-
-    async def __aenter__(self) -> Self:
-        wait = await asyncio.to_thread(self._reserve)
-        if wait > 0:
-            await asyncio.sleep(wait)
-        return self
-
-    async def __aexit__(self, *exc) -> bool:
-        return False
-
-
-_creation_limiters: dict[str, CreationLimiter] = {}
-
-
-def creation_limiter(per_sec: float | None, name: str) -> CreationLimiter | None:
-    """A user-global limiter pacing `name`'s creation to `per_sec`/s (None/<= 0 disables).
-
-    All callers (and processes) sharing a `name` share one bucket, so use one rate per name."""
-    if not per_sec or per_sec <= 0:
-        return None
-    limiter = _creation_limiters.get(name)
-    if limiter is None:
-        limiter = _creation_limiters[name] = CreationLimiter(name, per_sec)
-    return limiter

verifiers-0.3.2.dev127/verifiers/v1/tasksets/lean/__init__.py

@@ -1,33 +0,0 @@
-from verifiers.v1.tasksets.lean.scoring import (
-    build_starter_file,
-    expected_protected_signature,
-    parse_compile_output,
-    protected_signature_substring_present,
-    strip_lean_comments,
-)
-from verifiers.v1.tasksets.lean.taskset import (
-    DEFAULT_DOCKER_IMAGE,
-    LEAN_PROJECT_PATH,
-    PROOF_FILE_PATH,
-    LeanConfig,
-    LeanDatasetConfig,
-    LeanTask,
-    LeanTaskConfig,
-    LeanTaskset,
-)
-
-__all__ = [
-    "DEFAULT_DOCKER_IMAGE",
-    "LEAN_PROJECT_PATH",
-    "PROOF_FILE_PATH",
-    "LeanConfig",
-    "LeanDatasetConfig",
-    "LeanTask",
-    "LeanTaskConfig",
-    "LeanTaskset",
-    "build_starter_file",
-    "expected_protected_signature",
-    "parse_compile_output",
-    "protected_signature_substring_present",
-    "strip_lean_comments",
-]