verifiers 0.3.2.dev38__py3-none-any.whl → 0.3.2.dev40__py3-none-any.whl

verifiers/v1/cli/replay.py

@@ -156,7 +156,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
             else:
                 st.state = "running"
                 # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge", None)
+                trace.info.pop("judge_calls", None)
                 trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.

verifiers/v1/harnesses/prime_agent/harness.py

@@ -19,10 +19,10 @@ logger = logging.getLogger(__name__)
 GITHUB_RELEASE_URL = (
     "https://github.com/PrimeIntellect-ai/prime-agent/releases/download"
 )
-PRIME_AGENT_COMMIT: Literal["514633727bf26d74f39f3119c2b0e31a5ceb2a9d"] = (
-    "514633727bf26d74f39f3119c2b0e31a5ceb2a9d"
+PRIME_AGENT_COMMIT: Literal["81ae3cb34d27d38ee37f9e205a1e73694993b344"] = (
+    "81ae3cb34d27d38ee37f9e205a1e73694993b344"
 )
-PRIME_AGENT_VERSION = "0.8.1"
+PRIME_AGENT_VERSION = "0.9.1"
 PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent"
 STATE_ROOT = "/tmp/vf-prime-agent-runs"
 SKILLS_DIR = ".agents/skills"
@@ -51,10 +51,10 @@ for tarball in "$agent_tarball" "$ai_tarball" "$core_tarball" "$tui_tarball"; do
         "$release_url/$tarball" -o "$download_dir/$tarball"
 done
 printf '%s  %s\n' \
-    '46c24db1782dd31adc35d5c6cbcc75564faba6ced3bf2ccf03d836ee77134475' "$agent_tarball" \
-    'f6c3bdb6093bc24a327546fe865ef9a4a172c734fcd4c4093e30c19476f0134d' "$ai_tarball" \
-    '0cc3660953545f8ac9a7e704fcb9875f954d58c3085304080ef615c280aa5748' "$core_tarball" \
-    'bd07bccee0ca495565b1d62e9411f3fdebe49e3dfa52870564f08af5e61fde15' "$tui_tarball" \
+    '573bce0cd004fc62052e9a924089941b7f39266ab71e66a94c85a1f9d35835ba' "$agent_tarball" \
+    '11b5b4cf67b6bb2d3420a44fb69181bc9d94d81e69a2b4fde07eb9c99f5faf4f' "$ai_tarball" \
+    'fb6f3a5dcc8b69c5eeb3beff722b5e0f09885c14849db50bc1d7c0f1d064151c' "$core_tarball" \
+    '4f3eaca2814944d3993073e0b88c0bde54a641ddb5132fbe107e392b997e38ec' "$tui_tarball" \
     > "$download_dir/SHA256SUMS"
 (cd "$download_dir" && sha256sum -c SHA256SUMS)
 mkdir "$download_dir/package-root"
@@ -87,7 +87,7 @@ PRIME_AGENT_BOOTSTRAP_TOOLS_ON_INSTALL=1 npm install -g \
 
 
 class PrimeAgentHarnessConfig(HarnessConfig):
-    commit: Literal["514633727bf26d74f39f3119c2b0e31a5ceb2a9d"] = PRIME_AGENT_COMMIT
+    commit: Literal["81ae3cb34d27d38ee37f9e205a1e73694993b344"] = PRIME_AGENT_COMMIT
     """Prime Agent main commit to install."""
 
     autonomous: bool = False

verifiers/v1/judge.py

@@ -32,7 +32,8 @@ in which case `JudgeResponse.parsed` is the validated pydantic object.
 A judge is cheap to construct (the HTTP client is opened per call, inside `complete`, and
 closed when the call returns), so build it where you use it.
 
-Passing `trace=` records the call onto it — a typed record appended to `trace.info["judge"]` and
+Passing `trace=` records the call onto it — a request/response record appended to
+`trace.info["judge_calls"]` and
 the call's tokens + cost added to `trace.extra_usage` (kept separate from the agent's `trace.usage`),
 so judge behaviour and spend are no longer invisible. The record lands even if the judge refuses, an
 empty structured output comes back, or `parse` raises (the request was already billed). Omit `trace`
@@ -210,7 +211,14 @@ class Judge(Generic[ParsedT, ConfigT]):
             return response
         finally:
             if trace is not None and response is not None:
-                trace.record_judge(response)
+                trace.record_judge_call(
+                    name=self.reward_name,
+                    request={
+                        "model": kwargs["model"],
+                        "messages": kwargs["messages"],
+                    },
+                    response=response,
+                )
 
     async def evaluate(
         self, *, trace: Trace | None = None, **fields: Any

verifiers/v1/judges/rubric.py

@@ -24,7 +24,7 @@ RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text(
 
 # Appended to the prompt when `structured_output` is off, so the reply is JSON we can parse. Each
 # criterion carries a one-sentence `reason` written *before* the verdict (chain-of-thought), so the
-# verdict follows from it and the reasoning is auditable in `trace.info["judge"]`.
+# verdict follows from it and the reasoning is auditable in `trace.info["judge_calls"]`.
 JSON_SUFFIX = (
     "\n\nRespond with ONLY a JSON object and nothing else, in exactly this shape:\n"
     '{"verdicts": [{"name": "<criterion name>", "reason": "<one sentence citing specific '

verifiers/v1/trace.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import time
 import traceback
 import uuid
@@ -672,8 +673,26 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         reward = Reward(score=float(value), weight=float(weight))
         self.rewards[name] = reward
 
-    def record_judge(self, response: JudgeResponse) -> None:
-        self.info.setdefault("judge", []).append(response.model_dump())
+    def record_judge_call(
+        self,
+        *,
+        name: str,
+        request: Mapping[str, Any],
+        response: JudgeResponse,
+    ) -> None:
+        """Record one complete judge request/response exchange."""
+        response_record = response.model_dump()
+        message = AssistantMessage(content=response_record.pop("text"))
+        self.info.setdefault("judge_calls", []).append(
+            {
+                "name": name,
+                "request": copy.deepcopy(dict(request)),
+                "response": {
+                    "message": message.model_dump(),
+                    **response_record,
+                },
+            }
+        )
         if response.usage is not None:
             self.extra_usage.append(response.usage)
 

{verifiers-0.3.2.dev38.dist-info → verifiers-0.3.2.dev40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev38
+Version: 0.3.2.dev40
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev38.dist-info → verifiers-0.3.2.dev40.dist-info}/RECORD

@@ -6,14 +6,14 @@ verifiers/v1/episode.py,sha256=uStvI4WqgJT50hmHIB55m5B-ASqqLz-fZ-ynBz9uzEY,5474
 verifiers/v1/errors.py,sha256=uQySmCUrjDYFkBIii1vBFPiEcPFPQZ9EGpas_6HvcQI,5279
 verifiers/v1/graph.py,sha256=F07EZIti0AspEQ-0R0Cg_E8wJeKRlBHP4CCWytSL_8Q,35496
 verifiers/v1/harness.py,sha256=ziECqEBWV34UWbLlxrkhY6MwkbVYkoXBOoaY7mZxwTs,14753
-verifiers/v1/judge.py,sha256=Pvr0C41ah1qkNSZ0WXdkDvBMfv5YSfY1REHncwlJ8cg,9346
+verifiers/v1/judge.py,sha256=uqn-djkK2_7Q3kOjkDo6ipOvV_HOvuLa9y6AoDjRux8,9618
 verifiers/v1/rollout.py,sha256=hoKGn1CbShbyjEM9PgYq8pdKgoC91VebTkBSgUHLtiE,23978
 verifiers/v1/semantic.py,sha256=wssvBSEpY_tgzpFUADG9-N4HkDXgCZvNQ1pdf6qAsS0,4680
 verifiers/v1/session.py,sha256=xM7JJJVJACk0JfH9RAN1Ts8gJJKr2tGGRiIgQTrdieY,18692
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
 verifiers/v1/task.py,sha256=vaVkBu00riiNU1Ke4NeeY2NQTXxOguz9d6NLe55hO_Q,11468
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
-verifiers/v1/trace.py,sha256=bB-rp2Sqz3vM27QdcXJxcG9fAq0sQR3OAniLYO1-sJk,28808
+verifiers/v1/trace.py,sha256=-d4O06j3QG9dlv9nkWLupVjLt5-wqAf2GlXVgfBPcY8,29351
 verifiers/v1/types.py,sha256=PH1SxRWXOjPCgQD9aZgM18_bECvFSNQ25gkCbulmTKU,9857
 verifiers/v1/acp/__init__.py,sha256=wUPZVYBFjJuOnIIoYONBFuA04A1DJf_dOyKY3QwK37Q,12722
 verifiers/v1/acp/runner.py,sha256=gMXFmsVg2qD0Zhrx7kvC6Me2Z-wsfBamRIO0i45UW8g,11097
@@ -22,7 +22,7 @@ verifiers/v1/cli/debug.py,sha256=x3bqOwwbwwfwQQpxHrZeKdxsW3ZVtUT9cAn_r4aOro4,116
 verifiers/v1/cli/gepa.py,sha256=7YpppEMeuY2m1X5szGkybzTHztnIs83CvnSizhiEC-w,4210
 verifiers/v1/cli/init.py,sha256=LFF_1e2XTdgIKjWoqq_IAlOzAXcARPEroblcEGv0fCU,7599
 verifiers/v1/cli/output.py,sha256=HZbkHw662ppVdpTGutrKuQOSeBvtsAj5qjUokCkwkv0,8457
-verifiers/v1/cli/replay.py,sha256=JhhI_O8PLhtHbgLie5dG03RrTynvZtsEPX-MLY3KQy4,10240
+verifiers/v1/cli/replay.py,sha256=2lL7h6MrIn6hIh7fuqbw7PmruCxkK5iqwoeKebr-z7Q,10246
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
 verifiers/v1/cli/resume.py,sha256=hqV5AxxqeIInJeEZQerQznuijhMNijFEFcHZIltfLdE,470
 verifiers/v1/cli/validate.py,sha256=G3YXZTwxVDQIyR_ASapJceixOxRSJl69NnFPBqkF3N0,17173
@@ -107,7 +107,7 @@ verifiers/v1/harnesses/pi/harness.py,sha256=M0BRsfJxnBNP-KCrp0vT5gHlaJEDnyti5x_3
 verifiers/v1/harnesses/pool/__init__.py,sha256=HTYsNiWGdiNEsSHj4xhPmshJSNb7SHjQ7d4yZQ4p3HM,127
 verifiers/v1/harnesses/pool/harness.py,sha256=HKAogPbyLK5QpvPQ1teHOh_IyRQX-or2JF71nnZ_5Ps,3875
 verifiers/v1/harnesses/prime_agent/__init__.py,sha256=xfPC4RrlnyYWaf38tPAOZq4YL6M55lMpj5BoRLmaK8I,171
-verifiers/v1/harnesses/prime_agent/harness.py,sha256=byplbbIOFACCn2BvVZAxBSHdaPKBGERYz_hA1rgBj0E,11205
+verifiers/v1/harnesses/prime_agent/harness.py,sha256=_FJtiYvWOsBpjEBKuJsnmuldYNwJxKK8ONa5e-c_mYM,11205
 verifiers/v1/harnesses/rlm/__init__.py,sha256=hwx51xTdEWzzYgMj-p4ETNcptoub3IPMDbEdJ4uB8Ns,122
 verifiers/v1/harnesses/rlm/harness.py,sha256=1a88tUM8l4BRf-In-dPyhtW9ilyH2GJdbh1ezL2BRcQ,10297
 verifiers/v1/harnesses/terminus_2/__init__.py,sha256=l1pwfg1HyVDEqT9efdnFWcXqaZwtKwe7TrplkyxZSA0,166
@@ -130,7 +130,7 @@ verifiers/v1/interception/tunnel/prime.py,sha256=fIhU4Sa53f2FQWrwIgpDjh8XO9E9RdF
 verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
-verifiers/v1/judges/rubric.py,sha256=qwBjJCWfaUUcQA9A4EXFQK0NumGjoevSFT2HjgBZut4,12019
+verifiers/v1/judges/rubric.py,sha256=HGQYI3xdq8I84Ce0_sDeNgk5JM597es8rOVPipjCyxU,12025
 verifiers/v1/judges/rubric.txt,sha256=3KA2ZeYYdcDweJhUGopxu9EFgARhu2aWft5S5cj8Bc8,323
 verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,408
 verifiers/v1/mcp/launch.py,sha256=dqi5UCE-ShPn663glL4DwnTHk4_VBLG_0qUUwlr3p10,23776
@@ -185,8 +185,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev38.dist-info/METADATA,sha256=XtmwcXdAJSovO0iDRdqt2zmGCOe8Mh7oaGIxQNa2Euo,4065
-verifiers-0.3.2.dev38.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev38.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
-verifiers-0.3.2.dev38.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev38.dist-info/RECORD,,
+verifiers-0.3.2.dev40.dist-info/METADATA,sha256=e5bkm_Lm2AKE8KvhdjM6WmekaMIDNsqwuHT0ivu7QF0,4065
+verifiers-0.3.2.dev40.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev40.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
+verifiers-0.3.2.dev40.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev40.dist-info/RECORD,,