verifiers 0.3.2.dev203__tar.gz → 0.3.2.dev205__tar.gz

{verifiers-0.3.2.dev203 → verifiers-0.3.2.dev205}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev203
+Version: 0.3.2.dev205
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -37,7 +37,7 @@ Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-runs>=0.1.4
 Requires-Dist: prime-sandboxes>=0.4.2
-Requires-Dist: prime-tunnel>=0.1.13
+Requires-Dist: prime-tunnel>=0.1.14
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
 Requires-Dist: renderers[multimodal]>=0.1.12.dev2

{verifiers-0.3.2.dev203 → verifiers-0.3.2.dev205}/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "numpy>=2.1.0",
     "math-verify>=0.8.0",
     "openai>=2.54.0,<3.0.0",
-    "prime-tunnel>=0.1.13",
+    "prime-tunnel>=0.1.14",
     "prime-sandboxes>=0.4.2",
     "prime-runs>=0.1.4",
     "pydantic>=2.12.3",

verifiers-0.3.2.dev205/tests/v1/test_tunnel_watchdog.py

@@ -0,0 +1,109 @@
+"""The prime tunnel watchdog restarts frpc when the tunnel service has the tunnel
+disconnected or frpc has exited, leaves it alone otherwise, and stops watching a tunnel
+whose registration is gone."""
+
+import asyncio
+
+import pytest
+from prime_tunnel import TunnelGoneError, TunnelStatus
+
+from verifiers.v1.interception.tunnel import prime
+
+pytestmark = pytest.mark.asyncio
+
+GONE = (None, "terminated", "expired")
+
+
+class FakeClient:
+    def __init__(self, statuses, running=True, restart_error=None):
+        self.statuses = list(statuses)
+        self.is_running = running
+        self.is_gone = False
+        self.tunnel_id = "t-0-test"
+        self.restart_error = restart_error
+        self.restarts = 0
+        self.checks = 0
+
+    async def restart(self):
+        self.restarts += 1
+        if self.restart_error is not None:
+            raise self.restart_error
+        self.is_running = True
+
+    async def status(self):
+        self.checks += 1
+        registration = self.statuses.pop(0) if self.statuses else "connected"
+        if isinstance(registration, Exception):
+            raise registration
+        self.is_gone = self.is_gone or registration in GONE
+        return TunnelStatus(
+            tunnel_id=self.tunnel_id,
+            running=self.is_running,
+            registration=registration,
+            gone=self.is_gone,
+        )
+
+
+async def watch(client, checks, monkeypatch):
+    """Run the watchdog for about `checks` checks; how many restarts it made."""
+    monkeypatch.setattr(prime, "CHECK_SECONDS", 0.01)
+    task = asyncio.create_task(prime._watch(client, "https://t"))
+    await asyncio.sleep(0.0125 * checks)
+    task.cancel()
+    await asyncio.wait({task})
+    return client.restarts
+
+
+async def stops(client, monkeypatch):
+    """Run the watchdog until it gives the tunnel up."""
+    monkeypatch.setattr(prime, "CHECK_SECONDS", 0.01)
+    await asyncio.wait_for(prime._watch(client, "https://t"), 1)
+
+
+async def test_connected_is_left_alone(monkeypatch):
+    assert await watch(FakeClient(["connected"] * 10), 8, monkeypatch) == 0
+
+
+async def test_one_disconnected_check_is_not_enough(monkeypatch):
+    client = FakeClient(["disconnected", "connected"] * 5)
+    assert await watch(client, 8, monkeypatch) == 0
+
+
+async def test_disconnected_twice_restarts_once(monkeypatch):
+    client = FakeClient(["disconnected", "disconnected"])
+    assert await watch(client, 8, monkeypatch) == 1
+
+
+async def test_exited_frpc_restarts_without_asking(monkeypatch):
+    client = FakeClient([RuntimeError("api down")] * 10, running=False)
+    assert await watch(client, 4, monkeypatch) == 1
+
+
+async def test_unreachable_service_is_no_verdict(monkeypatch):
+    client = FakeClient([RuntimeError("api down")] * 10)
+    assert await watch(client, 8, monkeypatch) == 0
+
+
+@pytest.mark.parametrize("registration", GONE)
+async def test_gone_registration_stops_watching(registration, monkeypatch):
+    client = FakeClient([registration])
+    await stops(client, monkeypatch)
+    assert client.restarts == 0
+
+
+async def test_gone_reported_by_frpc_stops_watching(monkeypatch):
+    client = FakeClient([])
+    client.is_gone = True
+    await stops(client, monkeypatch)
+    assert (client.checks, client.restarts) == (0, 0)
+
+
+async def test_restart_refused_as_gone_stops_watching(monkeypatch):
+    client = FakeClient([], running=False, restart_error=TunnelGoneError("inactive"))
+    await stops(client, monkeypatch)
+    assert client.restarts == 1
+
+
+async def test_failed_restart_is_tried_again(monkeypatch):
+    client = FakeClient([], running=False, restart_error=RuntimeError("frpc"))
+    assert await watch(client, 6, monkeypatch) >= 2

{verifiers-0.3.2.dev203 → verifiers-0.3.2.dev205}/verifiers/v1/interception/tunnel/prime.py

@@ -53,22 +53,25 @@ CHECK_SECONDS = 20
 async def _watch(client: "TunnelClient", url: str) -> None:
     """Keep a held tunnel up. A tunnel service hiccup can leave frpc running while the
     service has it disconnected for good, and every request to the URL then hangs or
-    404s; frpc can also exit. Either way, restart it."""
+    404s; frpc can also exit. Either way, restart it. A tunnel the service has deleted
+    or expired can't be restarted, so stop watching it."""
+    from prime_tunnel import TunnelGoneError
+
     strikes = 0
     while True:
         await asyncio.sleep(CHECK_SECONDS * random.uniform(0.75, 1.25))
-        if client.is_running:
+        if client.is_running and not client.is_gone:
             try:
-                info = await client._client.get_tunnel(client.tunnel_id)
+                status = await client.status()
             except Exception as e:  # noqa: BLE001 - the service is unreachable: no verdict
                 logger.debug("tunnel %s: state check failed: %s", url, e)
                 continue
-            if info is None:
-                logger.error("tunnel %s: registration gone; cannot repair", url)
-                return
-            strikes = strikes + 1 if info.status == "disconnected" else 0
-            if strikes < 2:
-                continue
+            strikes = strikes + 1 if status.registration == "disconnected" else 0
+        if client.is_gone:
+            logger.error("tunnel %s: registration gone; cannot repair", url)
+            return
+        if client.is_running and strikes < 2:
+            continue
         logger.warning(
             "tunnel %s: %s; restarting frpc",
             url,
@@ -76,6 +79,9 @@ async def _watch(client: "TunnelClient", url: str) -> None:
         )
         try:
             await client.restart()
+        except TunnelGoneError:
+            logger.error("tunnel %s: registration gone; cannot repair", url)
+            return
         except Exception as e:  # noqa: BLE001 - tried again on the next check
             logger.warning("tunnel %s: frpc restart failed: %s", url, e)
         else:

verifiers-0.3.2.dev203/tests/v1/test_tunnel_watchdog.py

@@ -1,71 +0,0 @@
-"""The prime tunnel watchdog restarts frpc when the tunnel service has the tunnel
-disconnected or frpc has exited, and leaves it alone otherwise."""
-
-import asyncio
-from types import SimpleNamespace
-
-import pytest
-
-from verifiers.v1.interception.tunnel import prime
-
-pytestmark = pytest.mark.asyncio
-
-
-class FakeClient:
-    def __init__(self, statuses, running=True):
-        self.statuses = list(statuses)
-        self.is_running = running
-        self.tunnel_id = "t-0-test"
-        self._client = self
-        self.restarts = 0
-
-    async def restart(self):
-        self.restarts += 1
-        self.is_running = True
-
-    async def get_tunnel(self, tunnel_id):
-        status = self.statuses.pop(0) if self.statuses else "connected"
-        if isinstance(status, Exception):
-            raise status
-        return None if status is None else SimpleNamespace(status=status)
-
-
-async def watch(client, checks, monkeypatch):
-    """Run the watchdog for about `checks` checks; how many restarts it made."""
-    monkeypatch.setattr(prime, "CHECK_SECONDS", 0.01)
-    task = asyncio.create_task(prime._watch(client, "https://t"))
-    await asyncio.sleep(0.0125 * checks)
-    task.cancel()
-    await asyncio.wait({task})
-    return client.restarts
-
-
-async def test_connected_is_left_alone(monkeypatch):
-    assert await watch(FakeClient(["connected"] * 10), 8, monkeypatch) == 0
-
-
-async def test_one_disconnected_check_is_not_enough(monkeypatch):
-    client = FakeClient(["disconnected", "connected"] * 5)
-    assert await watch(client, 8, monkeypatch) == 0
-
-
-async def test_disconnected_twice_restarts_once(monkeypatch):
-    client = FakeClient(["disconnected", "disconnected"])
-    assert await watch(client, 8, monkeypatch) == 1
-
-
-async def test_exited_frpc_restarts_without_asking(monkeypatch):
-    client = FakeClient([RuntimeError("api down")] * 10, running=False)
-    assert await watch(client, 4, monkeypatch) == 1
-
-
-async def test_unreachable_service_is_no_verdict(monkeypatch):
-    client = FakeClient([RuntimeError("api down")] * 10)
-    assert await watch(client, 8, monkeypatch) == 0
-
-
-async def test_gone_registration_stops_watching(monkeypatch):
-    client = FakeClient([None])
-    monkeypatch.setattr(prime, "CHECK_SECONDS", 0.01)
-    await asyncio.wait_for(prime._watch(client, "https://t"), 1)
-    assert client.restarts == 0