verifiers 0.4.1.dev38__tar.gz → 0.4.1.dev40__tar.gz

{verifiers-0.4.1.dev38 → verifiers-0.4.1.dev40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev38
+Version: 0.4.1.dev40
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev38 → verifiers-0.4.1.dev40}/verifiers/v1/acp/__init__.py

@@ -27,6 +27,12 @@ from verifiers.v1.utils.aio import run_shielded
 
 ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text()
 MAX_PACKET_BYTES = 128 * 1024 * 1024
+_STOP_CONDITIONS = {
+    "max_tokens": "max_output_tokens",
+    "max_turn_requests": "max_turns",
+    "refusal": "refusal",
+    "cancelled": "cancelled",
+}
 
 __all__ = ["ACPConfig", "ACPHarness", "ACPTurn"]
 
@@ -354,6 +360,7 @@ class ACPHarnessSession(HarnessSession):
                 raise
         turn = _turn_result(response)
         self.trace.root_reply = turn.reply.strip()
+        self.trace.info["acp_stop_reason"] = turn.stop_reason
         if not response.get("ok"):
             detail = response.get("error") or "ACP session request failed"
             if stderr := self._stderr():
@@ -368,6 +375,8 @@ class ACPHarnessSession(HarnessSession):
         harness = cast(ACPHarness, self.harness)
         harness._consume_protocol_metadata(self.trace, turn.response_metadata)
         harness.acp_turn_result(self.trace, turn)
+        if condition := _STOP_CONDITIONS.get(turn.stop_reason or ""):
+            self.trace.stop(condition)
         result = ProgramResult(exit_code=0, stdout=turn.reply, stderr="")
         _require_model_turn(self.trace, calls_before, result)
         return result

{verifiers-0.4.1.dev38 → verifiers-0.4.1.dev40}/verifiers/v1/grading.py

@@ -23,6 +23,8 @@ from verifiers.v1.utils.env import resolve_env
 from verifiers.v1.utils.retries import RetryState, backoff
 
 logger = logging.getLogger(__name__)
+_SCORING_INFO_KEYS = ("judge_calls", "scoring_diagnostics")
+"""Accounting and diagnostics survive failed scoring attempts, unlike partial scores."""
 
 
 @contextmanager
@@ -32,11 +34,11 @@ def scoring_transaction(trace: Trace):
     try:
         yield
     except BaseException:
-        # Failed judges still consume tokens and calls; retain their accounting.
-        judge_calls = trace.info.get("judge_calls")
+        retained = {
+            key: trace.info[key] for key in _SCORING_INFO_KEYS if key in trace.info
+        }
         trace.state, trace.rewards, trace.metrics, trace.info = saved
-        if judge_calls is not None:
-            trace.info["judge_calls"] = judge_calls
+        trace.info.update(retained)
         raise
 
 
@@ -140,11 +142,16 @@ async def grade_task(
                         if on_score is not None:
                             await on_score(result, runtime)
                 finally:
-                    # Model-backed judges are billed even when an attempt fails.
-                    # Preserve their accounting without committing partial rewards/state.
+                    # Preserve failed attempts' evidence and accounting without
+                    # committing partial rewards/state.
                     trace.extra_usage = result.extra_usage
-                    if "judge_calls" in result.info:
-                        trace.info["judge_calls"] = result.info["judge_calls"]
+                    trace.info.update(
+                        {
+                            key: result.info[key]
+                            for key in _SCORING_INFO_KEYS
+                            if key in result.info
+                        }
+                    )
                 # Cleanup failure must not discard an authoritative completed score.
                 trace.state, trace.rewards, trace.metrics = (
                     result.state,

{verifiers-0.4.1.dev38 → verifiers-0.4.1.dev40}/verifiers/v1/tasksets/harbor/taskset.py

@@ -51,6 +51,7 @@ CACHE = Path.home() / ".cache" / "harbor"
 HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"
 REWARD_JSON = "/logs/verifier/reward.json"
 MAX_REWARD_BYTES = 1024 * 1024
+MAX_VERIFIER_OUTPUT_CHARS = 16 * 1024
 REWARD_JSON_ADAPTER = TypeAdapter(
     dict[str, float],
     config=ConfigDict(strict=True, allow_inf_nan=False),
@@ -447,6 +448,13 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
         result = await runtime.run(
             ["bash", "/tests/test.sh"], resolve_env(self.scoring.env or {})
         )
+        trace.info.setdefault("scoring_diagnostics", {})["harbor"] = {
+            "exit_code": result.exit_code,
+            "stdout": result.stdout[-MAX_VERIFIER_OUTPUT_CHARS:],
+            "stderr": result.stderr[-MAX_VERIFIER_OUTPUT_CHARS:],
+            "stdout_truncated": len(result.stdout) > MAX_VERIFIER_OUTPUT_CHARS,
+            "stderr_truncated": len(result.stderr) > MAX_VERIFIER_OUTPUT_CHARS,
+        }
         scores = await self.read_reward_json(runtime)
         if scores is None:
             try: