verifiers 0.3.2.dev200__tar.gz → 0.3.2.dev204__tar.gz

{verifiers-0.3.2.dev200 → verifiers-0.3.2.dev204}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev200
+Version: 0.3.2.dev204
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -37,7 +37,7 @@ Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-runs>=0.1.4
 Requires-Dist: prime-sandboxes>=0.4.2
-Requires-Dist: prime-tunnel>=0.1.12
+Requires-Dist: prime-tunnel>=0.1.13
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
 Requires-Dist: renderers[multimodal]>=0.1.12.dev2

{verifiers-0.3.2.dev200 → verifiers-0.3.2.dev204}/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "numpy>=2.1.0",
     "math-verify>=0.8.0",
     "openai>=2.54.0,<3.0.0",
-    "prime-tunnel>=0.1.12",
+    "prime-tunnel>=0.1.13",
     "prime-sandboxes>=0.4.2",
     "prime-runs>=0.1.4",
     "pydantic>=2.12.3",

verifiers-0.3.2.dev204/tests/v1/test_tunnel_watchdog.py

@@ -0,0 +1,71 @@
+"""The prime tunnel watchdog restarts frpc when the tunnel service has the tunnel
+disconnected or frpc has exited, and leaves it alone otherwise."""
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from verifiers.v1.interception.tunnel import prime
+
+pytestmark = pytest.mark.asyncio
+
+
+class FakeClient:
+    def __init__(self, statuses, running=True):
+        self.statuses = list(statuses)
+        self.is_running = running
+        self.tunnel_id = "t-0-test"
+        self._client = self
+        self.restarts = 0
+
+    async def restart(self):
+        self.restarts += 1
+        self.is_running = True
+
+    async def get_tunnel(self, tunnel_id):
+        status = self.statuses.pop(0) if self.statuses else "connected"
+        if isinstance(status, Exception):
+            raise status
+        return None if status is None else SimpleNamespace(status=status)
+
+
+async def watch(client, checks, monkeypatch):
+    """Run the watchdog for about `checks` checks; how many restarts it made."""
+    monkeypatch.setattr(prime, "CHECK_SECONDS", 0.01)
+    task = asyncio.create_task(prime._watch(client, "https://t"))
+    await asyncio.sleep(0.0125 * checks)
+    task.cancel()
+    await asyncio.wait({task})
+    return client.restarts
+
+
+async def test_connected_is_left_alone(monkeypatch):
+    assert await watch(FakeClient(["connected"] * 10), 8, monkeypatch) == 0
+
+
+async def test_one_disconnected_check_is_not_enough(monkeypatch):
+    client = FakeClient(["disconnected", "connected"] * 5)
+    assert await watch(client, 8, monkeypatch) == 0
+
+
+async def test_disconnected_twice_restarts_once(monkeypatch):
+    client = FakeClient(["disconnected", "disconnected"])
+    assert await watch(client, 8, monkeypatch) == 1
+
+
+async def test_exited_frpc_restarts_without_asking(monkeypatch):
+    client = FakeClient([RuntimeError("api down")] * 10, running=False)
+    assert await watch(client, 4, monkeypatch) == 1
+
+
+async def test_unreachable_service_is_no_verdict(monkeypatch):
+    client = FakeClient([RuntimeError("api down")] * 10)
+    assert await watch(client, 8, monkeypatch) == 0
+
+
+async def test_gone_registration_stops_watching(monkeypatch):
+    client = FakeClient([None])
+    monkeypatch.setattr(prime, "CHECK_SECONDS", 0.01)
+    await asyncio.wait_for(prime._watch(client, "https://t"), 1)
+    assert client.restarts == 0

{verifiers-0.3.2.dev200 → verifiers-0.3.2.dev204}/verifiers/v1/interception/tunnel/prime.py

@@ -6,8 +6,9 @@ pool scales with."""
 import asyncio
 import contextlib
 import logging
+import random
 from collections.abc import AsyncIterator
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 from verifiers.v1.interception.tunnel.base import BaseTunnelConfig, Tunnel
 from verifiers.v1.runtimes.limiters import CreationLimiter
@@ -15,6 +16,9 @@ from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.prime import ensure_prime_auth
 from verifiers.v1.utils.scope import run_scope
 
+if TYPE_CHECKING:
+    from prime_tunnel import Tunnel as TunnelClient
+
 logger = logging.getLogger(__name__)
 
 
@@ -40,6 +44,45 @@ def tunnel_limiter() -> CreationLimiter:
     return CreationLimiter("prime-tunnel", run_scope(), _TUNNELS_PER_MIN / 60)
 
 
+# How often a held tunnel's state is checked with the tunnel service. The service marks a
+# tunnel disconnected about a minute after its frpc stops answering; two such checks in a
+# row restart frpc.
+CHECK_SECONDS = 20
+
+
+async def _watch(client: "TunnelClient", url: str) -> None:
+    """Keep a held tunnel up. A tunnel service hiccup can leave frpc running while the
+    service has it disconnected for good, and every request to the URL then hangs or
+    404s; frpc can also exit. Either way, restart it."""
+    strikes = 0
+    while True:
+        await asyncio.sleep(CHECK_SECONDS * random.uniform(0.75, 1.25))
+        if client.is_running:
+            try:
+                info = await client._client.get_tunnel(client.tunnel_id)
+            except Exception as e:  # noqa: BLE001 - the service is unreachable: no verdict
+                logger.debug("tunnel %s: state check failed: %s", url, e)
+                continue
+            if info is None:
+                logger.error("tunnel %s: registration gone; cannot repair", url)
+                return
+            strikes = strikes + 1 if info.status == "disconnected" else 0
+            if strikes < 2:
+                continue
+        logger.warning(
+            "tunnel %s: %s; restarting frpc",
+            url,
+            "disconnected" if client.is_running else "frpc exited",
+        )
+        try:
+            await client.restart()
+        except Exception as e:  # noqa: BLE001 - tried again on the next check
+            logger.warning("tunnel %s: frpc restart failed: %s", url, e)
+        else:
+            logger.warning("tunnel %s: back up", url)
+            strikes = 0
+
+
 class PrimeTunnelConfig(BaseTunnelConfig):
     """Expose the host interception port via `prime_tunnel` (frpc). No fields — the tunnel
     service mints a fresh public URL per exposed port."""
@@ -57,7 +100,8 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
         """Bridge the host `port` to a public URL via prime_tunnel (frpc). Tunnel creation
         is network-bound and rate-capped (512/min, run-wide via the shared
         `tunnel_limiter`), so transient failures are retried; a terminal one raises
-        `TunnelError`. The tunnel is torn down on exit."""
+        `TunnelError`. While held, frpc is restarted if the tunnel goes down. The tunnel is
+        torn down on exit."""
         from prime_tunnel import Tunnel as TunnelClient
 
         from verifiers.v1.errors import TunnelError
@@ -72,10 +116,17 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
                         url = str(await client.start()).rstrip("/")
         except Exception as e:
             raise TunnelError(f"{label} failed: {e}") from e
+        watch = asyncio.create_task(_watch(client, url))
+
+        async def close() -> None:
+            watch.cancel()  # first, so it can't start a new frpc after the stop
+            await asyncio.wait({watch})
+            await asyncio.to_thread(client.sync_stop)
+
         try:
             yield url
         finally:
-            # Run the synchronous stop to completion even under cancellation (`run_shielded`
-            # re-raises the cancellation after); tunnel-stop failures are best-effort.
+            # Run the stop to completion even under cancellation (`run_shielded` re-raises
+            # the cancellation after); tunnel-stop failures are best-effort.
             with contextlib.suppress(Exception):
-                await run_shielded(asyncio.to_thread(client.sync_stop))
+                await run_shielded(close())