verifiers 0.4.1.dev37__tar.gz → 0.4.1.dev39__tar.gz

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev37
+Version: 0.4.1.dev39
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/verifiers/v1/acp/__init__.py

@@ -27,6 +27,12 @@ from verifiers.v1.utils.aio import run_shielded
 
 ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text()
 MAX_PACKET_BYTES = 128 * 1024 * 1024
+_STOP_CONDITIONS = {
+    "max_tokens": "max_output_tokens",
+    "max_turn_requests": "max_turns",
+    "refusal": "refusal",
+    "cancelled": "cancelled",
+}
 
 __all__ = ["ACPConfig", "ACPHarness", "ACPTurn"]
 
@@ -354,6 +360,7 @@ class ACPHarnessSession(HarnessSession):
                 raise
         turn = _turn_result(response)
         self.trace.root_reply = turn.reply.strip()
+        self.trace.info["acp_stop_reason"] = turn.stop_reason
         if not response.get("ok"):
             detail = response.get("error") or "ACP session request failed"
             if stderr := self._stderr():
@@ -368,6 +375,8 @@ class ACPHarnessSession(HarnessSession):
         harness = cast(ACPHarness, self.harness)
         harness._consume_protocol_metadata(self.trace, turn.response_metadata)
         harness.acp_turn_result(self.trace, turn)
+        if condition := _STOP_CONDITIONS.get(turn.stop_reason or ""):
+            self.trace.stop(condition)
         result = ProgramResult(exit_code=0, stdout=turn.reply, stderr="")
         _require_model_turn(self.trace, calls_before, result)
         return result

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/verifiers/v1/dialects/anthropic.py

@@ -64,6 +64,7 @@ ERROR_TYPES = {
 }
 ERROR_STATUSES = {kind: status for status, kind in ERROR_TYPES.items()}
 
+
 # Claude may reorder mixed thinking block types between a response and its replay.
 # Native tool events share the final rank, preserving their relative order.
 THINKING_ORDER = {"redacted_thinking": 0, "thinking": 1}
@@ -118,6 +119,12 @@ _SAFE_CONTENT_TYPES = (
 )
 
 
+def _error_type(status_code: int) -> str:
+    return ERROR_TYPES.get(
+        status_code, "api_error" if status_code >= 500 else "invalid_request_error"
+    )
+
+
 def parse_content(content) -> str | list[ContentPart]:
     """Anthropic user-side content (text + image blocks) -> typed content parts."""
     if isinstance(content, str):
@@ -525,10 +532,10 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
         # The SDK sends the key as `x-api-key`; an ANTHROPIC_AUTH_TOKEN arrives as Bearer.
         return headers.get("x-api-key") or super().secret(headers)
 
-    def error_body(self, message: str) -> dict:
+    def error_body(self, message: str, *, status_code: int = 400) -> dict:
         return {
             "type": "error",
-            "error": {"type": "invalid_request_error", "message": message},
+            "error": {"type": _error_type(status_code), "message": message},
         }
 
     def stream_keepalive(self, first: bool) -> bytes:
@@ -538,10 +545,7 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
     def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
         # Once HTTP 200 is committed, clients classify retries from the SSE error type.
         if status_code is not None:
-            kind = ERROR_TYPES.get(
-                status_code,
-                "api_error" if status_code >= 500 else "invalid_request_error",
-            )
+            kind = _error_type(status_code)
             error = {**error, "error": {**error.get("error", {}), "type": kind}}
         # The Anthropic SDKs raise only on a named `error` event.
         return b"event: error\ndata: " + json.dumps(error).encode() + b"\n\n"

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/verifiers/v1/dialects/base.py

@@ -316,7 +316,7 @@ class Dialect(ABC, Generic[RespT]):
         client ends on an earlier event (e.g. Responses' `response.completed`) overrides this."""
         return is_sse_done_event(chunk)
 
-    def error_body(self, message: str) -> dict:
+    def error_body(self, message: str, *, status_code: int = 400) -> dict:
         """An error payload in this format's error shape (OpenAI by default)."""
         return {"error": {"message": message, "type": "invalid_request_error"}}
 

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/verifiers/v1/grading.py

@@ -23,6 +23,8 @@ from verifiers.v1.utils.env import resolve_env
 from verifiers.v1.utils.retries import RetryState, backoff
 
 logger = logging.getLogger(__name__)
+_SCORING_INFO_KEYS = ("judge_calls", "scoring_diagnostics")
+"""Accounting and diagnostics survive failed scoring attempts, unlike partial scores."""
 
 
 @contextmanager
@@ -32,11 +34,11 @@ def scoring_transaction(trace: Trace):
     try:
         yield
     except BaseException:
-        # Failed judges still consume tokens and calls; retain their accounting.
-        judge_calls = trace.info.get("judge_calls")
+        retained = {
+            key: trace.info[key] for key in _SCORING_INFO_KEYS if key in trace.info
+        }
         trace.state, trace.rewards, trace.metrics, trace.info = saved
-        if judge_calls is not None:
-            trace.info["judge_calls"] = judge_calls
+        trace.info.update(retained)
         raise
 
 
@@ -140,11 +142,16 @@ async def grade_task(
                         if on_score is not None:
                             await on_score(result, runtime)
                 finally:
-                    # Model-backed judges are billed even when an attempt fails.
-                    # Preserve their accounting without committing partial rewards/state.
+                    # Preserve failed attempts' evidence and accounting without
+                    # committing partial rewards/state.
                     trace.extra_usage = result.extra_usage
-                    if "judge_calls" in result.info:
-                        trace.info["judge_calls"] = result.info["judge_calls"]
+                    trace.info.update(
+                        {
+                            key: result.info[key]
+                            for key in _SCORING_INFO_KEYS
+                            if key in result.info
+                        }
+                    )
                 # Cleanup failure must not discard an authoritative completed score.
                 trace.state, trace.rewards, trace.metrics = (
                     result.state,

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/verifiers/v1/interception/server.py

@@ -114,6 +114,13 @@ async def _request_digest(raw: bytes) -> bytes:
     return await asyncio.to_thread(_body_digest, raw)
 
 
+def _error_response(dialect: Dialect, message: str, *, status: int) -> web.Response:
+    """Keep the native error classification consistent with its HTTP status."""
+    return web.json_response(
+        dialect.error_body(message, status_code=status), status=status
+    )
+
+
 def _completion_response(completion: dict | None) -> web.Response:
     """Serialize a model's JSON-native response without an intermediate string."""
     try:
@@ -470,9 +477,8 @@ class InterceptionServer(Interception):
         logger.warning(
             "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error
         )
-        return web.json_response(
-            dialect.error_body(str(error)),
-            status=getattr(error, "status_code", 502),
+        return _error_response(
+            dialect, str(error), status=getattr(error, "status_code", 502)
         )
 
     def mediate_capabilities(
@@ -589,7 +595,7 @@ class InterceptionServer(Interception):
         session = self.sessions.get(dialect.secret(request.headers))
         if session is None:
             logger.warning("interception: unauthorized request")
-            return web.json_response(dialect.error_body("unauthorized"), status=401)
+            return _error_response(dialect, "unauthorized", status=401)
         session.adopt(asyncio.current_task())
         started = time.monotonic()
         try:
@@ -621,7 +627,7 @@ class InterceptionServer(Interception):
         try:
             acp, upstream_headers = extract_acp_info(request.headers)
         except ValueError as error:
-            return web.json_response(dialect.error_body(str(error)), status=400)
+            return _error_response(dialect, str(error), status=400)
         logger.debug(
             "intercept %s: id=%s stream=%s retry=%s",
             request.path,
@@ -655,10 +661,9 @@ class InterceptionServer(Interception):
         if idempotency_key or retried:
             idempotent = session.idempotent_requests.get(replay_key)
         if idempotent is not None and idempotent.binding != binding:
-            return web.json_response(
-                dialect.error_body(
-                    "Idempotency-Key was reused with a different request"
-                ),
+            return _error_response(
+                dialect,
+                "Idempotency-Key was reused with a different request",
                 status=400,
             )
         if idempotent is not None and idempotent.response is not None:
@@ -671,15 +676,12 @@ class InterceptionServer(Interception):
         try:
             model_request = dialect.parse_request(body)
         except ValueError as error:
-            return web.json_response(dialect.error_body(str(error)), status=400)
+            return _error_response(dialect, str(error), status=400)
         if session.released:
-            return web.json_response(
-                dialect.error_body("rollout concluded"), status=409
-            )
+            return _error_response(dialect, "rollout concluded", status=409)
         if session.stopped:
-            return web.json_response(
-                dialect.error_body(f"rollout stopped: {session.trace.stop_condition}"),
-                status=400,
+            return _error_response(
+                dialect, f"rollout stopped: {session.trace.stop_condition}", status=400
             )
         if idempotent is None:
             idempotent = IdempotentRequest(binding=binding)
@@ -693,9 +695,7 @@ class InterceptionServer(Interception):
             )
             response = await asyncio.shield(inflight)
             if response is None:
-                return web.json_response(
-                    dialect.error_body("upstream attempt failed"), status=503
-                )
+                return _error_response(dialect, "upstream attempt failed", status=503)
             return _replay_response(response)
 
         if idempotent.inflight is not None:
@@ -718,8 +718,8 @@ class InterceptionServer(Interception):
         try:
             refused = await session.refused()
             if refused is not None:
-                return web.json_response(
-                    dialect.error_body(f"rollout stopped: {refused}"), status=400
+                return _error_response(
+                    dialect, f"rollout stopped: {refused}", status=400
                 )
             original_request = model_request
             model_request, request_rewrites, stopped = await session.rewrite_request(
@@ -747,10 +747,7 @@ class InterceptionServer(Interception):
             )
             turn.commit_prompt()
             session.trace.stop(stopped)
-            return web.json_response(
-                dialect.error_body(f"rollout stopped: {stopped}"),
-                status=400,
-            )
+            return _error_response(dialect, f"rollout stopped: {stopped}", status=400)
 
         try:
             body, policy_paths = self.mediate_capabilities(session, dialect, body)
@@ -761,7 +758,7 @@ class InterceptionServer(Interception):
                 session.trace, model_request.messages, model_request.tools, dialect.name
             )
         except ValueError as error:
-            return web.json_response(dialect.error_body(str(error)), status=400)
+            return _error_response(dialect, str(error), status=400)
         except RolloutError as error:
             return self._fail(session, dialect, error)
         # The tail is what the harness added since the last turn (tool results, user
@@ -822,9 +819,7 @@ class InterceptionServer(Interception):
                         len(call_response.message.tool_calls or []),
                     )
                     if session.released:  # concluded while sampling — seal holds
-                        return web.json_response(
-                            dialect.error_body("rollout concluded"), status=409
-                        )
+                        return _error_response(dialect, "rollout concluded", status=409)
                     response_rewrites = []
                     stopped = None
                     if session.response_interceptors or session.response_stops:
@@ -845,10 +840,9 @@ class InterceptionServer(Interception):
                             )
                             call_response.raw = raw_response
                     if session.stopped:
-                        return web.json_response(
-                            dialect.error_body(
-                                f"rollout stopped: {session.trace.stop_condition}"
-                            ),
+                        return _error_response(
+                            dialect,
+                            f"rollout stopped: {session.trace.stop_condition}",
                             status=400,
                         )
                     node = turn.commit(call_response)
@@ -858,9 +852,8 @@ class InterceptionServer(Interception):
                         stopped = await session.gate_tool_calls(node)
                     if stopped is not None:
                         session.trace.stop(stopped)
-                        return web.json_response(
-                            dialect.error_body(f"rollout stopped: {stopped}"),
-                            status=400,
+                        return _error_response(
+                            dialect, f"rollout stopped: {stopped}", status=400
                         )
                 except RolloutError as e:
                     # Stash the real cause; the rollout re-raises it after the harness returns.
@@ -873,9 +866,8 @@ class InterceptionServer(Interception):
                         type(e).__name__,
                         e,
                     )
-                    return web.json_response(
-                        dialect.error_body(str(e)),
-                        status=getattr(e, "status_code", 502),
+                    return _error_response(
+                        dialect, str(e), status=getattr(e, "status_code", 502)
                     )
                 except Exception as e:  # noqa: BLE001 - surface as an API error
                     error = e
@@ -885,7 +877,7 @@ class InterceptionServer(Interception):
                         type(e).__name__,
                         e,
                     )
-                    return web.json_response(dialect.error_body(str(e)), status=502)
+                    return _error_response(dialect, str(e), status=502)
                 except BaseException as e:
                     # A cancelled exchange (harness disconnect, shutdown) is still
                     # recorded, coupled to its cancellation.
@@ -923,7 +915,7 @@ class InterceptionServer(Interception):
         relayed as native JSON, never recorded on the trace."""
         session = self.sessions.get(dialect.secret(request.headers))
         if session is None:
-            return web.json_response(dialect.error_body("unauthorized"), status=401)
+            return _error_response(dialect, "unauthorized", status=401)
         session.adopt(asyncio.current_task())
         logger.debug("intercept aux %s: id=%s", route, session.trace.id)
         try:
@@ -942,12 +934,12 @@ class InterceptionServer(Interception):
                 type(e).__name__,
                 e,
             )
-            return web.json_response(
-                dialect.error_body(str(e)), status=getattr(e, "status_code", 502)
+            return _error_response(
+                dialect, str(e), status=getattr(e, "status_code", 502)
             )
         except Exception as e:  # noqa: BLE001 - surface auxiliary relay failures
             logger.warning("aux call failed: id=%s %s", session.trace.id, e)
-            return web.json_response(dialect.error_body(str(e)), status=502)
+            return _error_response(dialect, str(e), status=502)
         return web.json_response(result)
 
     async def handle_models(self, request: web.Request) -> web.Response:
@@ -978,7 +970,7 @@ class InterceptionServer(Interception):
                 )
         except httpx.HTTPError as e:
             logger.warning("models call failed: id=%s %s", session.trace.id, e)
-            return web.json_response(dialect.error_body(str(e)), status=502)
+            return _error_response(dialect, str(e), status=502)
         return web.Response(
             body=upstream.content,
             status=upstream.status_code,

{verifiers-0.4.1.dev37 → verifiers-0.4.1.dev39}/verifiers/v1/tasksets/harbor/taskset.py

@@ -51,6 +51,7 @@ CACHE = Path.home() / ".cache" / "harbor"
 HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"
 REWARD_JSON = "/logs/verifier/reward.json"
 MAX_REWARD_BYTES = 1024 * 1024
+MAX_VERIFIER_OUTPUT_CHARS = 16 * 1024
 REWARD_JSON_ADAPTER = TypeAdapter(
     dict[str, float],
     config=ConfigDict(strict=True, allow_inf_nan=False),
@@ -447,6 +448,13 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
         result = await runtime.run(
             ["bash", "/tests/test.sh"], resolve_env(self.scoring.env or {})
         )
+        trace.info.setdefault("scoring_diagnostics", {})["harbor"] = {
+            "exit_code": result.exit_code,
+            "stdout": result.stdout[-MAX_VERIFIER_OUTPUT_CHARS:],
+            "stderr": result.stderr[-MAX_VERIFIER_OUTPUT_CHARS:],
+            "stdout_truncated": len(result.stdout) > MAX_VERIFIER_OUTPUT_CHARS,
+            "stderr_truncated": len(result.stderr) > MAX_VERIFIER_OUTPUT_CHARS,
+        }
         scores = await self.read_reward_json(runtime)
         if scores is None:
             try: