verifiers 0.3.2.dev133__py3-none-any.whl → 0.3.2.dev134__py3-none-any.whl

verifiers/v1/runtimes/prime.py

@@ -35,6 +35,16 @@ logger = logging.getLogger(__name__)
 EFFECTIVELY_UNBOUNDED_SECONDS = 30 * 24 * 60 * 60
 """Safety deadline for APIs that require a finite bound. Normal execution remains
 bounded by idle detection or rollout cancellation; 30 days is above any real run."""
+# The SDK reads finished jobs' output through a bounded pool whose deadline includes time
+# spent queued; its defaults (20 reads, 45s) expire most reads when thousands of rollouts
+# finish a job at once (e.g. every harness's setup at a 3k-rollout start).
+_OUTPUT_READS = 100
+"""Concurrent output reads on the shared client."""
+_OUTPUT_DEADLINE_SECONDS = 300
+"""Deadline for one read of a finished job's output, queue time included."""
+_OUTPUT_RETRIES = 10
+"""Re-reads of a finished job's output that the SDK still failed to fetch, before the
+exec is reported as failed."""
 
 
 BASE_LABELS: list[str] = []
@@ -150,7 +160,9 @@ class PrimeRuntime(Runtime):
         loop = asyncio.get_running_loop()
         shared = _shared_clients.get(loop)
         if shared is None:
-            shared = _shared_clients[loop] = _SharedClient(AsyncSandboxClient())
+            shared = _shared_clients[loop] = _SharedClient(
+                AsyncSandboxClient(background_job_output_concurrency=_OUTPUT_READS)
+            )
         shared.leases += 1
         self._client = shared.client
         # Map the resources onto prime's API (minutes, split GPU; memory/disk are already
@@ -279,18 +291,34 @@ class PrimeRuntime(Runtime):
                 env=self.process_env(env),
             )
             delay = 0.1
+            output_retries = 0
+            missing = None
             while True:
-                result = await self._client.get_background_job(self.info.id, job)
-                if result.completed:
+                result = await self._client.get_background_job(
+                    self.info.id, job, timeout=_OUTPUT_DEADLINE_SECONDS
+                )
+                # Under load the SDK can see a job finish yet miss its output (its bounded
+                # output reads expire while queued) and reports that as `*_error`. The job
+                # is done, so asking again re-reads the output.
+                missing = result.stdout_error or result.stderr_error
+                if result.completed and (
+                    missing is None or output_retries == _OUTPUT_RETRIES
+                ):
                     break
+                if result.completed:
+                    output_retries += 1
                 await asyncio.sleep(delay)
                 delay = min(delay * 2, 3)
         except (
             Exception
         ) as e:  # a sandbox/API failure is one rollout's problem, not the eval's
             raise SandboxError(f"prime exec failed: {e}") from e
+        if missing is not None:
+            raise SandboxError(f"prime exec output unavailable: {missing}")
+        if result.exit_code is None:
+            raise SandboxError("prime exec completed without an exit code")
         return ProgramResult(
-            exit_code=result.exit_code or 0,
+            exit_code=result.exit_code,
             stdout=result.stdout or "",
             stderr=result.stderr or "",
         )

{verifiers-0.3.2.dev133.dist-info → verifiers-0.3.2.dev134.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev133
+Version: 0.3.2.dev134
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev133.dist-info → verifiers-0.3.2.dev134.dist-info}/RECORD

@@ -145,7 +145,7 @@ verifiers/v1/runtimes/base.py,sha256=IKmrJSJltWX9M1nPUZExKluPJM4N82VsnqQGFz0ECV8
 verifiers/v1/runtimes/container.py,sha256=puQD8X9acP6GefoCNEnq40POi-nXPD-729bUjTVShfM,9554
 verifiers/v1/runtimes/limiters.py,sha256=rOfQJcMdYcLSKq8AL9O09BfGOLqpYmNgrCxXXNcHLmU,2849
 verifiers/v1/runtimes/modal.py,sha256=6ccitC0HVsAP_l8soJ_RQYNIVKrJjDsz2vAJBtxUZjw,17476
-verifiers/v1/runtimes/prime.py,sha256=nTtu-uBjc39UxfMPEyndshfAiIdk1V4L4aE6zSVVSn0,17673
+verifiers/v1/runtimes/prime.py,sha256=Ly7eTEJY06-kjBBZhpJVALtsW23M-ZLJ1mlq27uSS9U,19158
 verifiers/v1/runtimes/subprocess.py,sha256=QmpQ235_xrX6gowrr8557VgbydOXpLbfFq0KKGfJThk,8774
 verifiers/v1/runtimes/docker/__init__.py,sha256=2wceP-8V-w0dQBEeLtoJFfCZ135DxP2rcdcUBmSWiNU,18543
 verifiers/v1/runtimes/docker/egress.py,sha256=wtKWTL2_zFidxMR7lDRmfevQlOsl2Iyk88ebdO4Kqqw,24566
@@ -190,8 +190,8 @@ verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4
 verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,838
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev133.dist-info/METADATA,sha256=Ff2GNRZ7xB_YdcKn5Cu6_FLNmVH6PuNJYVXZ-brj6mE,4161
-verifiers-0.3.2.dev133.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev133.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.3.2.dev133.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev133.dist-info/RECORD,,
+verifiers-0.3.2.dev134.dist-info/METADATA,sha256=CEOjkwaS7mTLz0q7Z8Bw-4TsRp-2HwXgtegTv3ps5Os,4161
+verifiers-0.3.2.dev134.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev134.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
+verifiers-0.3.2.dev134.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev134.dist-info/RECORD,,