verifiers 0.3.2.dev39__py3-none-any.whl → 0.3.2.dev40__py3-none-any.whl

verifiers/v1/cli/replay.py

@@ -156,7 +156,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
             else:
                 st.state = "running"
                 # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge", None)
+                trace.info.pop("judge_calls", None)
                 trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.

verifiers/v1/judge.py

@@ -32,7 +32,8 @@ in which case `JudgeResponse.parsed` is the validated pydantic object.
 A judge is cheap to construct (the HTTP client is opened per call, inside `complete`, and
 closed when the call returns), so build it where you use it.
 
-Passing `trace=` records the call onto it — a typed record appended to `trace.info["judge"]` and
+Passing `trace=` records the call onto it — a request/response record appended to
+`trace.info["judge_calls"]` and
 the call's tokens + cost added to `trace.extra_usage` (kept separate from the agent's `trace.usage`),
 so judge behaviour and spend are no longer invisible. The record lands even if the judge refuses, an
 empty structured output comes back, or `parse` raises (the request was already billed). Omit `trace`
@@ -210,7 +211,14 @@ class Judge(Generic[ParsedT, ConfigT]):
             return response
         finally:
             if trace is not None and response is not None:
-                trace.record_judge(response)
+                trace.record_judge_call(
+                    name=self.reward_name,
+                    request={
+                        "model": kwargs["model"],
+                        "messages": kwargs["messages"],
+                    },
+                    response=response,
+                )
 
     async def evaluate(
         self, *, trace: Trace | None = None, **fields: Any

verifiers/v1/judges/rubric.py

@@ -24,7 +24,7 @@ RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text(
 
 # Appended to the prompt when `structured_output` is off, so the reply is JSON we can parse. Each
 # criterion carries a one-sentence `reason` written *before* the verdict (chain-of-thought), so the
-# verdict follows from it and the reasoning is auditable in `trace.info["judge"]`.
+# verdict follows from it and the reasoning is auditable in `trace.info["judge_calls"]`.
 JSON_SUFFIX = (
     "\n\nRespond with ONLY a JSON object and nothing else, in exactly this shape:\n"
     '{"verdicts": [{"name": "<criterion name>", "reason": "<one sentence citing specific '

verifiers/v1/trace.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import time
 import traceback
 import uuid
@@ -672,8 +673,26 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         reward = Reward(score=float(value), weight=float(weight))
         self.rewards[name] = reward
 
-    def record_judge(self, response: JudgeResponse) -> None:
-        self.info.setdefault("judge", []).append(response.model_dump())
+    def record_judge_call(
+        self,
+        *,
+        name: str,
+        request: Mapping[str, Any],
+        response: JudgeResponse,
+    ) -> None:
+        """Record one complete judge request/response exchange."""
+        response_record = response.model_dump()
+        message = AssistantMessage(content=response_record.pop("text"))
+        self.info.setdefault("judge_calls", []).append(
+            {
+                "name": name,
+                "request": copy.deepcopy(dict(request)),
+                "response": {
+                    "message": message.model_dump(),
+                    **response_record,
+                },
+            }
+        )
         if response.usage is not None:
             self.extra_usage.append(response.usage)
 

{verifiers-0.3.2.dev39.dist-info → verifiers-0.3.2.dev40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev39
+Version: 0.3.2.dev40
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev39.dist-info → verifiers-0.3.2.dev40.dist-info}/RECORD

@@ -6,14 +6,14 @@ verifiers/v1/episode.py,sha256=uStvI4WqgJT50hmHIB55m5B-ASqqLz-fZ-ynBz9uzEY,5474
 verifiers/v1/errors.py,sha256=uQySmCUrjDYFkBIii1vBFPiEcPFPQZ9EGpas_6HvcQI,5279
 verifiers/v1/graph.py,sha256=F07EZIti0AspEQ-0R0Cg_E8wJeKRlBHP4CCWytSL_8Q,35496
 verifiers/v1/harness.py,sha256=ziECqEBWV34UWbLlxrkhY6MwkbVYkoXBOoaY7mZxwTs,14753
-verifiers/v1/judge.py,sha256=Pvr0C41ah1qkNSZ0WXdkDvBMfv5YSfY1REHncwlJ8cg,9346
+verifiers/v1/judge.py,sha256=uqn-djkK2_7Q3kOjkDo6ipOvV_HOvuLa9y6AoDjRux8,9618
 verifiers/v1/rollout.py,sha256=hoKGn1CbShbyjEM9PgYq8pdKgoC91VebTkBSgUHLtiE,23978
 verifiers/v1/semantic.py,sha256=wssvBSEpY_tgzpFUADG9-N4HkDXgCZvNQ1pdf6qAsS0,4680
 verifiers/v1/session.py,sha256=xM7JJJVJACk0JfH9RAN1Ts8gJJKr2tGGRiIgQTrdieY,18692
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
 verifiers/v1/task.py,sha256=vaVkBu00riiNU1Ke4NeeY2NQTXxOguz9d6NLe55hO_Q,11468
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
-verifiers/v1/trace.py,sha256=bB-rp2Sqz3vM27QdcXJxcG9fAq0sQR3OAniLYO1-sJk,28808
+verifiers/v1/trace.py,sha256=-d4O06j3QG9dlv9nkWLupVjLt5-wqAf2GlXVgfBPcY8,29351
 verifiers/v1/types.py,sha256=PH1SxRWXOjPCgQD9aZgM18_bECvFSNQ25gkCbulmTKU,9857
 verifiers/v1/acp/__init__.py,sha256=wUPZVYBFjJuOnIIoYONBFuA04A1DJf_dOyKY3QwK37Q,12722
 verifiers/v1/acp/runner.py,sha256=gMXFmsVg2qD0Zhrx7kvC6Me2Z-wsfBamRIO0i45UW8g,11097
@@ -22,7 +22,7 @@ verifiers/v1/cli/debug.py,sha256=x3bqOwwbwwfwQQpxHrZeKdxsW3ZVtUT9cAn_r4aOro4,116
 verifiers/v1/cli/gepa.py,sha256=7YpppEMeuY2m1X5szGkybzTHztnIs83CvnSizhiEC-w,4210
 verifiers/v1/cli/init.py,sha256=LFF_1e2XTdgIKjWoqq_IAlOzAXcARPEroblcEGv0fCU,7599
 verifiers/v1/cli/output.py,sha256=HZbkHw662ppVdpTGutrKuQOSeBvtsAj5qjUokCkwkv0,8457
-verifiers/v1/cli/replay.py,sha256=JhhI_O8PLhtHbgLie5dG03RrTynvZtsEPX-MLY3KQy4,10240
+verifiers/v1/cli/replay.py,sha256=2lL7h6MrIn6hIh7fuqbw7PmruCxkK5iqwoeKebr-z7Q,10246
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
 verifiers/v1/cli/resume.py,sha256=hqV5AxxqeIInJeEZQerQznuijhMNijFEFcHZIltfLdE,470
 verifiers/v1/cli/validate.py,sha256=G3YXZTwxVDQIyR_ASapJceixOxRSJl69NnFPBqkF3N0,17173
@@ -130,7 +130,7 @@ verifiers/v1/interception/tunnel/prime.py,sha256=fIhU4Sa53f2FQWrwIgpDjh8XO9E9RdF
 verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
-verifiers/v1/judges/rubric.py,sha256=qwBjJCWfaUUcQA9A4EXFQK0NumGjoevSFT2HjgBZut4,12019
+verifiers/v1/judges/rubric.py,sha256=HGQYI3xdq8I84Ce0_sDeNgk5JM597es8rOVPipjCyxU,12025
 verifiers/v1/judges/rubric.txt,sha256=3KA2ZeYYdcDweJhUGopxu9EFgARhu2aWft5S5cj8Bc8,323
 verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,408
 verifiers/v1/mcp/launch.py,sha256=dqi5UCE-ShPn663glL4DwnTHk4_VBLG_0qUUwlr3p10,23776
@@ -185,8 +185,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev39.dist-info/METADATA,sha256=aCN9__3iHq_uJT58Pv_szAvF_zQiUkj2j_IihmNAi3I,4065
-verifiers-0.3.2.dev39.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev39.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
-verifiers-0.3.2.dev39.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev39.dist-info/RECORD,,
+verifiers-0.3.2.dev40.dist-info/METADATA,sha256=e5bkm_Lm2AKE8KvhdjM6WmekaMIDNsqwuHT0ivu7QF0,4065
+verifiers-0.3.2.dev40.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev40.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
+verifiers-0.3.2.dev40.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev40.dist-info/RECORD,,