verifiers 0.2.2.dev22__tar.gz → 0.2.2.dev23__tar.gz

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev22
+Version: 0.2.2.dev23
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev23}/verifiers/v1/push.py

@@ -6,6 +6,7 @@ contract as `prime eval push`, done inline at the end of a run. Auth + base URL
 come from `$PRIME_API_KEY` / `~/.prime/config.json`.
 """
 
+import json
 import logging
 import os
 from dataclasses import dataclass
@@ -22,6 +23,8 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_API_URL = "https://api.primeintellect.ai"
 DEFAULT_FRONTEND_URL = "https://app.primeintellect.ai"
+# Repeated /samples posts append; match the Prime Evals client's request ceiling.
+_MAX_SAMPLES_PAYLOAD_BYTES = 25 * 1024 * 1024
 
 
 @dataclass
@@ -200,6 +203,36 @@ def push_traces(
     # The run is done and its results saved; a network blip here must not crash it
     # — log and skip the upload instead.
     try:
+        batches: list[list[dict[str, Any]]] = []
+        batch: list[dict[str, Any]] = []
+        payload_bytes = len(b'{"samples":[]}')
+        for i, sample in enumerate(samples):
+            sample_bytes = len(
+                json.dumps(
+                    sample,
+                    ensure_ascii=False,
+                    separators=(",", ":"),
+                    allow_nan=False,
+                ).encode("utf-8")
+            )
+            sample_payload_bytes = len(b'{"samples":[]}') + sample_bytes
+            if sample_payload_bytes > _MAX_SAMPLES_PAYLOAD_BYTES:
+                raise ValueError(
+                    f"sample {i} is too large to upload "
+                    f"({sample_payload_bytes} > "
+                    f"{_MAX_SAMPLES_PAYLOAD_BYTES} bytes)"
+                )
+            next_payload_bytes = payload_bytes + (1 if batch else 0) + sample_bytes
+            if batch and next_payload_bytes > _MAX_SAMPLES_PAYLOAD_BYTES:
+                batches.append(batch)
+                batch = []
+                payload_bytes = len(b'{"samples":[]}')
+                next_payload_bytes = payload_bytes + sample_bytes
+            batch.append(sample)
+            payload_bytes = next_payload_bytes
+        if batch or not samples:
+            batches.append(batch)
+
         with httpx.Client(headers=headers, timeout=300.0) as client:
 
             def post(path: str, body: dict) -> dict:
@@ -224,7 +257,18 @@ def push_traces(
                     **team,
                 },
             )["evaluation_id"]
-            post(f"/evaluations/{eval_id}/samples", {"samples": samples})
+            for batch in batches:
+                body = json.dumps(
+                    {"samples": batch},
+                    ensure_ascii=False,
+                    separators=(",", ":"),
+                    allow_nan=False,
+                ).encode("utf-8")
+                resp = client.post(
+                    f"{api}/evaluations/{eval_id}/samples",
+                    content=body,
+                )
+                resp.raise_for_status()
             post(f"/evaluations/{eval_id}/finalize", {"metrics": metrics})
     except Exception as e:
         logger.warning("--push: upload failed (%s: %s); skipping", type(e).__name__, e)