verifiers 0.2.2.dev93__tar.gz → 0.2.2.dev95__tar.gz

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev93
+Version: 0.2.2.dev95
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/tests/v1/test_e2e.py

@@ -231,11 +231,12 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     assert trace.ok, trace.errors
     assert trace.stop_condition == "user_closed"
     assert trace.rewards["resumed"].score == 1.0
-    assert trace.tools  # ACP-native tools, or Pi's MCP adapter meta-tool
     segments = trace.info["acp_segments"]
     assert len(segments) == 2
     assert segments[0]["terminated"] is False
     assert segments[1]["terminated"] is False
+    # Native MCP tools need not appear in the intercepted model request that
+    # populates trace.tools; the ACP transcript is the source of truth for use.
     assert "tool" in segments[1]["roles"]
     assert segments[1]["tool_outputs"]
     if harness == "rlm":

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/envs/experimental/composable/harnesses/rlm.py

@@ -15,6 +15,7 @@ from verifiers.envs.experimental.utils.git_checkout_cache import (
     validate_git_checkout,
 )
 from verifiers.types import Messages, State, SystemMessage, TrajectoryStep
+from verifiers.utils.path_utils import CACHE_DIR
 
 DEFAULT_RLM_REPO_URL = "github.com/PrimeIntellect-ai/rlm-harness.git"
 DEFAULT_RLM_REF = "main"
@@ -24,9 +25,7 @@ DEFAULT_RLM_MAX_DEPTH = 0
 DEFAULT_APPEND_TO_SYSTEM_PROMPT_PATH = "/task/append_to_system_prompt.txt"
 DEFAULT_RLM_CHECKOUT_PATH = "/tmp/rlm-checkout"
 DEFAULT_RLM_CHECKOUT_UPLOAD_NAME = "rlm_checkout"
-DEFAULT_RLM_LOCAL_CHECKOUT_CACHE_ROOT = (
-    Path.home() / ".cache" / "verifiers" / "rlm-checkouts"
-)
+DEFAULT_RLM_LOCAL_CHECKOUT_CACHE_ROOT = CACHE_DIR / "rlm-checkouts"
 _REQUIRED_CHECKOUT_FILES = ("install.sh", "pyproject.toml")
 
 COMPACTION_BOUNDARY_MARKER = "--- context compacted ---"

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/envs/experimental/utils/git_checkout_cache.py

@@ -16,6 +16,7 @@ from verifiers.envs.experimental.utils.file_locks import (
     exclusive_path_lock,
     sibling_lock_path,
 )
+from verifiers.utils.path_utils import CACHE_DIR
 
 _IN_USE_LOCK_SUFFIX = ".in-use.lock"
 
@@ -23,7 +24,7 @@ _IN_USE_LOCK_SUFFIX = ".in-use.lock"
 # resolved checkout's in-use lock file. See ``_acquire_in_use_lock``.
 _held_in_use_locks: dict[Path, IO] = {}
 
-DEFAULT_GIT_CHECKOUT_CACHE_ROOT = Path.home() / ".cache" / "verifiers" / "git-checkouts"
+DEFAULT_GIT_CHECKOUT_CACHE_ROOT = CACHE_DIR / "git-checkouts"
 _FULL_COMMIT_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$")
 
 logger = logging.getLogger(__name__)

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/utils/path_utils.py

@@ -9,6 +9,18 @@ from verifiers.types import EvalConfig
 logger = logging.getLogger(__name__)
 
 
+def home_dir() -> Path:
+    """Best-effort home directory; fall back to the temp dir so import never fails."""
+    try:
+        return Path.home()
+    except RuntimeError:
+        return Path(tempfile.gettempdir())
+
+
+CACHE_DIR = home_dir() / ".cache" / "verifiers"
+"""User-local cache root for verifiers-managed state."""
+
+
 def write_temp_file(content: str, suffix: str = ".txt") -> str:
     """Write content to a named temporary file and return its path.
 

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/v1/acp/runner.py

@@ -36,6 +36,7 @@ from acp.schema import (
 )
 
 MAX_PACKET_BYTES = 128 * 1024 * 1024
+LATE_UPDATE_GRACE_SECONDS = 1.0
 
 
 class VerifiersACPClient(Client):
@@ -43,6 +44,7 @@ class VerifiersACPClient(Client):
         self.visible_reply = ""
         self.message_id: str | None = None
         self.tool_calls: dict[str, str] = {}
+        self.output_changed = asyncio.Condition()
 
     def reset(self) -> None:
         self.visible_reply = ""
@@ -50,22 +52,23 @@ class VerifiersACPClient(Client):
         self.tool_calls = {}
 
     async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None:
-        if isinstance(update, ToolCall):
-            self.tool_calls[update.tool_call_id] = update.status or "pending"
-            return
-        if isinstance(update, ToolCallUpdate):
-            if update.status:
-                self.tool_calls[update.tool_call_id] = update.status
-            return
-        if not isinstance(update, AgentMessageChunk) or not isinstance(
-            update.content, TextContentBlock
-        ):
-            return
-        message_id = getattr(update, "message_id", None)
-        if message_id is not None and message_id != self.message_id:
-            self.visible_reply = ""
-            self.message_id = message_id
-        self.visible_reply += update.content.text
+        async with self.output_changed:
+            if isinstance(update, ToolCall):
+                self.tool_calls[update.tool_call_id] = update.status or "pending"
+            elif isinstance(update, ToolCallUpdate):
+                if update.status:
+                    self.tool_calls[update.tool_call_id] = update.status
+            elif isinstance(update, AgentMessageChunk) and isinstance(
+                update.content, TextContentBlock
+            ):
+                message_id = getattr(update, "message_id", None)
+                if message_id is not None and message_id != self.message_id:
+                    self.visible_reply = ""
+                    self.message_id = message_id
+                self.visible_reply += update.content.text
+            else:
+                return
+            self.output_changed.notify_all()
 
     async def request_permission(
         self,
@@ -172,6 +175,25 @@ async def prompt(
     except RequestError as error:
         detail = error.data.get("details") if isinstance(error.data, dict) else None
         raise RuntimeError(detail or str(error)) from error
+
+    # ACP 0.11 dispatches notifications in background tasks but resolves a request
+    # response directly in its receive loop. An agent that sends its final
+    # session/update immediately before session/prompt returns can therefore wake
+    # this coroutine before the update handler has run. Wait specifically for text:
+    # a completed tool update may also arrive first and must not hide a later reply.
+    def has_visible_reply() -> bool:
+        return bool(client.visible_reply.strip())
+
+    if not has_visible_reply():
+        async with client.output_changed:
+            try:
+                await asyncio.wait_for(
+                    client.output_changed.wait_for(has_visible_reply),
+                    timeout=LATE_UPDATE_GRACE_SECONDS,
+                )
+            except asyncio.TimeoutError:  # noqa: UP041 - Python 3.10 compatibility
+                pass
+
     tool_statuses = list(client.tool_calls.values())
     completed_tool_turn = (
         config.get("allow_empty_tool_reply", False)
@@ -179,7 +201,7 @@ async def prompt(
         and bool(tool_statuses)
         and all(status in ("completed", "failed") for status in tool_statuses)
     )
-    if not client.visible_reply.strip() and not completed_tool_turn:
+    if not has_visible_reply() and not completed_tool_turn:
         raise RuntimeError(
             "ACP agent produced no visible reply "
             f"(stop_reason={response.stop_reason}, tool_statuses={tool_statuses})"

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/v1/interception/tunnel/prime.py

@@ -13,7 +13,7 @@ from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
 
 # The prime_tunnel service caps tunnel starts at 512/min per API token — a property of the
-# tunnel service, shared by every process on the host that opens one. One host-global
+# tunnel service, shared by every process for the user that opens one. One user-global
 # limiter, not a per-runtime config knob.
 _TUNNELS_PER_MIN = 512
 TUNNEL_LIMITER = creation_limiter(_TUNNELS_PER_MIN / 60, "prime-tunnel")
@@ -30,7 +30,7 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
     @contextlib.asynccontextmanager
     async def expose(self, port: int) -> AsyncIterator[str]:
         """Bridge the host `port` to a public URL via prime_tunnel (frpc). Tunnel creation
-        is network-bound and globally rate-capped (512/min, host-wide via the shared
+        is network-bound and globally rate-capped (512/min, user-wide via the shared
         `TUNNEL_LIMITER`), so transient failures are retried; a terminal one raises
         `TunnelError`. The tunnel is torn down on exit."""
         from prime_tunnel import Tunnel as TunnelClient

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/v1/runtimes/limiters.py

@@ -1,41 +1,45 @@
-"""Host-global creation-rate limiters for the remote runtimes.
+"""User-global creation-rate limiters for the remote runtimes.
 
-A leaky bucket backed by a lock file, so a provider's per-account creation rate (Modal
-sandboxes, Prime tunnels) is enforced across EVERY process on the host — the single-process
-eval and all the elastically-spawned env-server worker processes alike — not just within one
-process. So the configured rate is the actual account-wide rate (assuming one env-server
-host). Keyed by name: one bucket file per name, shared by every process (and run) on the host.
+A leaky bucket backed by a lock file under the user cache (``~/.cache/verifiers``, falling
+back to the temp dir when no home is resolvable), so a provider's per-account creation rate
+(Modal sandboxes, Prime tunnels) is enforced across EVERY process for the user —
+the single-process eval and all the elastically-spawned env-server worker processes alike — not
+just within one process. Keyed by name: one bucket file per name, shared by every process (and
+run) for the user.
 """
 
 import asyncio
 import fcntl
 import os
-import tempfile
 import time
 from typing import Self
 
-_LIMITER_DIR = os.path.join(tempfile.gettempdir(), "vf-rate-limiters")
+from verifiers.utils.path_utils import CACHE_DIR
+
+LIMITER_DIR = CACHE_DIR / "limiter"
 
 
 class CreationLimiter:
     """An async leaky bucket shared across processes via a lock file: each `async with`
     reserves the next `1/per_sec`-spaced slot (advancing the on-disk cursor under an exclusive
-    flock) and sleeps until it, so the aggregate creation rate across all host processes stays
-    at `per_sec`. The reservation runs off the event loop; the wait does not hold the lock."""
+    flock) and sleeps until it, so the aggregate creation rate across all of the user's
+    processes stays at `per_sec`. The reservation runs off the event loop; the wait does not
+    hold the lock."""
 
     def __init__(self, name: str, per_sec: float) -> None:
         self._interval = 1 / per_sec
-        self._path = os.path.join(_LIMITER_DIR, f"{name}.bucket")
+        self._path = LIMITER_DIR / f"{name}.bucket"
 
     def _reserve(self) -> float:
-        os.makedirs(_LIMITER_DIR, exist_ok=True)
-        # monotonic is host-wide on Linux, so the cursor is comparable across processes.
+        os.makedirs(LIMITER_DIR, exist_ok=True)
+        # Wall clock persists across reboots (monotonic does not), so a cursor left in
+        # ~/.cache from a previous boot cannot turn into a huge stale wait.
         with open(self._path, "a+") as f:
             fcntl.flock(f.fileno(), fcntl.LOCK_EX)
             try:
                 f.seek(0)
                 data = f.read().strip()
-                now = time.monotonic()
+                now = time.time()
                 slot = max(now, float(data) if data else 0.0)
                 f.seek(0)
                 f.truncate()
@@ -59,7 +63,7 @@ _creation_limiters: dict[str, CreationLimiter] = {}
 
 
 def creation_limiter(per_sec: float | None, name: str) -> CreationLimiter | None:
-    """A host-global limiter pacing `name`'s creation to `per_sec`/s (None/<= 0 disables).
+    """A user-global limiter pacing `name`'s creation to `per_sec`/s (None/<= 0 disables).
 
     All callers (and processes) sharing a `name` share one bucket, so use one rate per name."""
     if not per_sec or per_sec <= 0:

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/v1/runtimes/modal.py

@@ -53,7 +53,7 @@ class ModalConfig(BaseConfig):
     """Disk in GB. Modal sandboxes have no disk knob, so this is accepted (so a task can
     declare it without a warning) but not enforced."""
     creates_per_sec: float | None = 40.0
-    """Pace sandbox creation to this many per second, enforced host-wide across every
+    """Pace sandbox creation to this many per second, enforced user-wide across every
     env-server worker process (None/<= 0 disables it)."""
 
 

{verifiers-0.2.2.dev93 → verifiers-0.2.2.dev95}/verifiers/v1/runtimes/prime.py

@@ -66,7 +66,7 @@ class PrimeConfig(NetworkPolicyConfig):
     idle_timeout: float | None = 3600
     """Seconds of inactivity before the sandbox self-deletes (None disables)."""
     creates_per_min: int | None = None
-    """Pace sandbox creation to this many per minute, enforced host-wide across every
+    """Pace sandbox creation to this many per minute, enforced user-wide across every
     env-server worker process (None/<= 0 disables it). (Tunnel creation is limited separately
     and globally — see interception.tunnel.prime.TUNNEL_LIMITER.)"""