verifiers 0.3.2.dev39__tar.gz → 0.3.2.dev40__tar.gz

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev39
+Version: 0.3.2.dev40
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/tests/v1/test_e2e.py

@@ -404,7 +404,7 @@ async def test_rubric_judge(run_v1, tmp_path):
     assert trace.ok
     assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
     assert trace.metrics["rubric/always_yes"] == 1.0
-    assert trace.info["judge"]  # the call was recorded onto the trace
+    assert trace.info["judge_calls"]  # the call was recorded onto the trace
 
 
 @pytest.mark.e2e
@@ -701,11 +701,29 @@ async def test_replay_round_trip(run_v1, tmp_path):
     assert source.ok
     assert "lcs" in source.rewards
 
+    # Replay must not mix the source run's judge transcript into newly computed
+    # judge calls. Seed a saved call directly because this task scores without a
+    # judge; cleanup happens before task scoring and therefore does not inspect it.
+    import json
+
+    stream = run_dir / "traces.jsonl"
+    record = json.loads(stream.read_text())
+    record["traces"][0]["info"]["judge_calls"] = [
+        {
+            "name": "source-judge",
+            "request": {"model": "source-model", "messages": []},
+            "response": {
+                "message": {"role": "assistant", "content": "stale"},
+                "parsed": None,
+                "usage": None,
+            },
+        }
+    ]
+    stream.write_text(json.dumps(record) + "\n")
+
     async def replay(source_dir: Path, out: Path):
         # The CLI's layering, minus the argv plumbing: the saved run's config is the base
         # (`ReplayConfig` ignores its eval-only keys), the source's output_dir is dropped.
-        import json
-
         data = json.loads(saved_config_path(source_dir).read_text())
         data.pop("output_dir", None)
         config = ReplayConfig(**{**data, "rich": False})
@@ -720,6 +738,7 @@ async def test_replay_round_trip(run_v1, tmp_path):
         # and recomputed the same value.
         assert replayed.rewards.keys() == source.rewards.keys()
         assert replayed.reward == pytest.approx(source.reward)
+        assert "judge_calls" not in replayed.info
     # The wire task keeps its taskset-specific fields in the replay's own output.
     raw = (tmp_path / "replay2" / "traces.jsonl").read_text()
     assert '"answer"' in raw

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/tests/v1/test_judges.py

@@ -92,7 +92,20 @@ def fake_judge_model(monkeypatch):
         if parse is not None:
             response.parsed = parse(response)
         if trace is not None:
-            trace.record_judge(response)
+            from verifiers.v1.dialects.chat import message_to_wire
+
+            wire = (
+                [{"role": "user", "content": messages}]
+                if isinstance(messages, str)
+                else [message_to_wire(message) for message in messages]
+            )
+            kwargs = {"model": self.config.model, "messages": wire}
+            kwargs.update(sampling)
+            trace.record_judge_call(
+                name=self.reward_name,
+                request={"model": kwargs["model"], "messages": kwargs["messages"]},
+                response=response,
+            )
         return response
 
     monkeypatch.setattr(Judge, "complete", fake_complete)
@@ -225,7 +238,43 @@ async def test_reference_score(fake_judge_model):
     assert (
         "Capital of France?" in fake_judge_model[0]
     )  # the task prompt is in the judge prompt
-    assert len(trace.info["judge"]) == 1  # the call is recorded onto the trace
+    assert len(trace.info["judge_calls"]) == 1  # the call is recorded onto the trace
+    judge_record = trace.info["judge_calls"][0]
+    assert judge_record["name"] == "reference"
+    assert judge_record["request"]["model"] == "openai/gpt-5.4-nano"
+    assert "Capital of France?" in judge_record["request"]["messages"][0]["content"]
+    assert judge_record["response"]["message"] == {
+        "role": "assistant",
+        "content": "yes",
+        "reasoning_content": None,
+        "tool_calls": None,
+        "provider_state": None,
+    }
+    assert judge_record["response"]["parsed"] == 1.0
+
+    request_messages = [{"role": "user", "content": "original"}]
+    request = {
+        "model": "openai/gpt-5.4-nano",
+        "messages": request_messages,
+    }
+    trace.record_judge_call(
+        name="snapshot",
+        request=request,
+        response=JudgeResponse(text="yes"),
+    )
+    request_messages[0]["content"] = "mutated"
+    assert (
+        trace.info["judge_calls"][-1]["request"]["messages"][0]["content"] == "original"
+    )
+
+    override_trace = make_trace()
+    await vf.ReferenceJudge().complete(
+        "Judge this response.", trace=override_trace, model="openai/gpt-5.4-mini"
+    )
+    assert (
+        override_trace.info["judge_calls"][0]["request"]["model"]
+        == "openai/gpt-5.4-mini"
+    )
 
     trace = make_trace(reply="It is Rome.")
     assert await vf.ReferenceJudge().score(trace.task.data, trace) == 0.0
@@ -446,7 +495,14 @@ async def test_error_attribution(monkeypatch, tmp_path):
             return response
         finally:
             if trace is not None:
-                trace.record_judge(response)
+                trace.record_judge_call(
+                    name=self.reward_name,
+                    request={
+                        "model": self.config.model,
+                        "messages": [{"role": "user", "content": messages}],
+                    },
+                    response=response,
+                )
 
     monkeypatch.setattr(Judge, "complete", gibberish_judge)
     taskset = JudgedTaskset(
@@ -463,7 +519,7 @@ async def test_error_attribution(monkeypatch, tmp_path):
             trace, runtime=None
         )
     assert trace.rewards["reference"] is None  # seeded: expected but never scored
-    assert len(trace.info["judge"]) == 1  # the billed call is still recorded
+    assert len(trace.info["judge_calls"]) == 1  # the billed call is still recorded
 
 
 # --- rubric --------------------------------------------------------------------------------
@@ -542,7 +598,7 @@ async def test_rubric_score(tmp_path, fake_judge_model):
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 0.75
     assert trace.metrics == {"rubric/mentions_paris": 1.0, "rubric/is_polite": 0.0}
-    assert len(trace.info["judge"]) == 1  # one call for the whole rubric
+    assert len(trace.info["judge_calls"]) == 1  # one call for the whole rubric
 
 
 async def test_rubric_verdict_mismatch_raises(tmp_path, monkeypatch):
@@ -652,7 +708,7 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
         trace.rewards["quality"].score == 0.75
     )  # the rubric's aggregate, under its `name`
     assert (
-        len(trace.info["judge"]) == 2
+        len(trace.info["judge_calls"]) == 2
     )  # every judge call recorded (rubric = one call)
 
 

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/verifiers/v1/cli/replay.py

@@ -156,7 +156,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
             else:
                 st.state = "running"
                 # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge", None)
+                trace.info.pop("judge_calls", None)
                 trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/verifiers/v1/judge.py

@@ -32,7 +32,8 @@ in which case `JudgeResponse.parsed` is the validated pydantic object.
 A judge is cheap to construct (the HTTP client is opened per call, inside `complete`, and
 closed when the call returns), so build it where you use it.
 
-Passing `trace=` records the call onto it — a typed record appended to `trace.info["judge"]` and
+Passing `trace=` records the call onto it — a request/response record appended to
+`trace.info["judge_calls"]` and
 the call's tokens + cost added to `trace.extra_usage` (kept separate from the agent's `trace.usage`),
 so judge behaviour and spend are no longer invisible. The record lands even if the judge refuses, an
 empty structured output comes back, or `parse` raises (the request was already billed). Omit `trace`
@@ -210,7 +211,14 @@ class Judge(Generic[ParsedT, ConfigT]):
             return response
         finally:
             if trace is not None and response is not None:
-                trace.record_judge(response)
+                trace.record_judge_call(
+                    name=self.reward_name,
+                    request={
+                        "model": kwargs["model"],
+                        "messages": kwargs["messages"],
+                    },
+                    response=response,
+                )
 
     async def evaluate(
         self, *, trace: Trace | None = None, **fields: Any

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/verifiers/v1/judges/rubric.py

@@ -24,7 +24,7 @@ RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text(
 
 # Appended to the prompt when `structured_output` is off, so the reply is JSON we can parse. Each
 # criterion carries a one-sentence `reason` written *before* the verdict (chain-of-thought), so the
-# verdict follows from it and the reasoning is auditable in `trace.info["judge"]`.
+# verdict follows from it and the reasoning is auditable in `trace.info["judge_calls"]`.
 JSON_SUFFIX = (
     "\n\nRespond with ONLY a JSON object and nothing else, in exactly this shape:\n"
     '{"verdicts": [{"name": "<criterion name>", "reason": "<one sentence citing specific '

{verifiers-0.3.2.dev39 → verifiers-0.3.2.dev40}/verifiers/v1/trace.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import time
 import traceback
 import uuid
@@ -672,8 +673,26 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         reward = Reward(score=float(value), weight=float(weight))
         self.rewards[name] = reward
 
-    def record_judge(self, response: JudgeResponse) -> None:
-        self.info.setdefault("judge", []).append(response.model_dump())
+    def record_judge_call(
+        self,
+        *,
+        name: str,
+        request: Mapping[str, Any],
+        response: JudgeResponse,
+    ) -> None:
+        """Record one complete judge request/response exchange."""
+        response_record = response.model_dump()
+        message = AssistantMessage(content=response_record.pop("text"))
+        self.info.setdefault("judge_calls", []).append(
+            {
+                "name": name,
+                "request": copy.deepcopy(dict(request)),
+                "response": {
+                    "message": message.model_dump(),
+                    **response_record,
+                },
+            }
+        )
         if response.usage is not None:
             self.extra_usage.append(response.usage)