verifiers 0.3.2.dev133__tar.gz → 0.3.2.dev134__tar.gz

{verifiers-0.3.2.dev133 → verifiers-0.3.2.dev134}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev133
+Version: 0.3.2.dev134
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev133 → verifiers-0.3.2.dev134}/verifiers/v1/runtimes/prime.py

@@ -35,6 +35,16 @@ logger = logging.getLogger(__name__)
 EFFECTIVELY_UNBOUNDED_SECONDS = 30 * 24 * 60 * 60
 """Safety deadline for APIs that require a finite bound. Normal execution remains
 bounded by idle detection or rollout cancellation; 30 days is above any real run."""
+# The SDK reads finished jobs' output through a bounded pool whose deadline includes time
+# spent queued; its defaults (20 reads, 45s) expire most reads when thousands of rollouts
+# finish a job at once (e.g. every harness's setup at a 3k-rollout start).
+_OUTPUT_READS = 100
+"""Concurrent output reads on the shared client."""
+_OUTPUT_DEADLINE_SECONDS = 300
+"""Deadline for one read of a finished job's output, queue time included."""
+_OUTPUT_RETRIES = 10
+"""Re-reads of a finished job's output that the SDK still failed to fetch, before the
+exec is reported as failed."""
 
 
 BASE_LABELS: list[str] = []
@@ -150,7 +160,9 @@ class PrimeRuntime(Runtime):
         loop = asyncio.get_running_loop()
         shared = _shared_clients.get(loop)
         if shared is None:
-            shared = _shared_clients[loop] = _SharedClient(AsyncSandboxClient())
+            shared = _shared_clients[loop] = _SharedClient(
+                AsyncSandboxClient(background_job_output_concurrency=_OUTPUT_READS)
+            )
         shared.leases += 1
         self._client = shared.client
         # Map the resources onto prime's API (minutes, split GPU; memory/disk are already
@@ -279,18 +291,34 @@ class PrimeRuntime(Runtime):
                 env=self.process_env(env),
             )
             delay = 0.1
+            output_retries = 0
+            missing = None
             while True:
-                result = await self._client.get_background_job(self.info.id, job)
-                if result.completed:
+                result = await self._client.get_background_job(
+                    self.info.id, job, timeout=_OUTPUT_DEADLINE_SECONDS
+                )
+                # Under load the SDK can see a job finish yet miss its output (its bounded
+                # output reads expire while queued) and reports that as `*_error`. The job
+                # is done, so asking again re-reads the output.
+                missing = result.stdout_error or result.stderr_error
+                if result.completed and (
+                    missing is None or output_retries == _OUTPUT_RETRIES
+                ):
                     break
+                if result.completed:
+                    output_retries += 1
                 await asyncio.sleep(delay)
                 delay = min(delay * 2, 3)
         except (
             Exception
         ) as e:  # a sandbox/API failure is one rollout's problem, not the eval's
             raise SandboxError(f"prime exec failed: {e}") from e
+        if missing is not None:
+            raise SandboxError(f"prime exec output unavailable: {missing}")
+        if result.exit_code is None:
+            raise SandboxError("prime exec completed without an exit code")
         return ProgramResult(
-            exit_code=result.exit_code or 0,
+            exit_code=result.exit_code,
             stdout=result.stdout or "",
             stderr=result.stderr or "",
         )