verifiers 0.3.2.dev36__tar.gz → 0.3.2.dev41__tar.gz

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev36
+Version: 0.3.2.dev41
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/tests/v1/test_e2e.py

@@ -404,7 +404,7 @@ async def test_rubric_judge(run_v1, tmp_path):
     assert trace.ok
     assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
     assert trace.metrics["rubric/always_yes"] == 1.0
-    assert trace.info["judge"]  # the call was recorded onto the trace
+    assert trace.info["judge_calls"]  # the call was recorded onto the trace
 
 
 @pytest.mark.e2e
@@ -701,11 +701,29 @@ async def test_replay_round_trip(run_v1, tmp_path):
     assert source.ok
     assert "lcs" in source.rewards
 
+    # Replay must not mix the source run's judge transcript into newly computed
+    # judge calls. Seed a saved call directly because this task scores without a
+    # judge; cleanup happens before task scoring and therefore does not inspect it.
+    import json
+
+    stream = run_dir / "traces.jsonl"
+    record = json.loads(stream.read_text())
+    record["traces"][0]["info"]["judge_calls"] = [
+        {
+            "name": "source-judge",
+            "request": {"model": "source-model", "messages": []},
+            "response": {
+                "message": {"role": "assistant", "content": "stale"},
+                "parsed": None,
+                "usage": None,
+            },
+        }
+    ]
+    stream.write_text(json.dumps(record) + "\n")
+
     async def replay(source_dir: Path, out: Path):
         # The CLI's layering, minus the argv plumbing: the saved run's config is the base
         # (`ReplayConfig` ignores its eval-only keys), the source's output_dir is dropped.
-        import json
-
         data = json.loads(saved_config_path(source_dir).read_text())
         data.pop("output_dir", None)
         config = ReplayConfig(**{**data, "rich": False})
@@ -720,6 +738,7 @@ async def test_replay_round_trip(run_v1, tmp_path):
         # and recomputed the same value.
         assert replayed.rewards.keys() == source.rewards.keys()
         assert replayed.reward == pytest.approx(source.reward)
+        assert "judge_calls" not in replayed.info
     # The wire task keeps its taskset-specific fields in the replay's own output.
     raw = (tmp_path / "replay2" / "traces.jsonl").read_text()
     assert '"answer"' in raw

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/tests/v1/test_judges.py

@@ -92,7 +92,20 @@ def fake_judge_model(monkeypatch):
         if parse is not None:
             response.parsed = parse(response)
         if trace is not None:
-            trace.record_judge(response)
+            from verifiers.v1.dialects.chat import message_to_wire
+
+            wire = (
+                [{"role": "user", "content": messages}]
+                if isinstance(messages, str)
+                else [message_to_wire(message) for message in messages]
+            )
+            kwargs = {"model": self.config.model, "messages": wire}
+            kwargs.update(sampling)
+            trace.record_judge_call(
+                name=self.reward_name,
+                request={"model": kwargs["model"], "messages": kwargs["messages"]},
+                response=response,
+            )
         return response
 
     monkeypatch.setattr(Judge, "complete", fake_complete)
@@ -225,7 +238,43 @@ async def test_reference_score(fake_judge_model):
     assert (
         "Capital of France?" in fake_judge_model[0]
     )  # the task prompt is in the judge prompt
-    assert len(trace.info["judge"]) == 1  # the call is recorded onto the trace
+    assert len(trace.info["judge_calls"]) == 1  # the call is recorded onto the trace
+    judge_record = trace.info["judge_calls"][0]
+    assert judge_record["name"] == "reference"
+    assert judge_record["request"]["model"] == "openai/gpt-5.4-nano"
+    assert "Capital of France?" in judge_record["request"]["messages"][0]["content"]
+    assert judge_record["response"]["message"] == {
+        "role": "assistant",
+        "content": "yes",
+        "reasoning_content": None,
+        "tool_calls": None,
+        "provider_state": None,
+    }
+    assert judge_record["response"]["parsed"] == 1.0
+
+    request_messages = [{"role": "user", "content": "original"}]
+    request = {
+        "model": "openai/gpt-5.4-nano",
+        "messages": request_messages,
+    }
+    trace.record_judge_call(
+        name="snapshot",
+        request=request,
+        response=JudgeResponse(text="yes"),
+    )
+    request_messages[0]["content"] = "mutated"
+    assert (
+        trace.info["judge_calls"][-1]["request"]["messages"][0]["content"] == "original"
+    )
+
+    override_trace = make_trace()
+    await vf.ReferenceJudge().complete(
+        "Judge this response.", trace=override_trace, model="openai/gpt-5.4-mini"
+    )
+    assert (
+        override_trace.info["judge_calls"][0]["request"]["model"]
+        == "openai/gpt-5.4-mini"
+    )
 
     trace = make_trace(reply="It is Rome.")
     assert await vf.ReferenceJudge().score(trace.task.data, trace) == 0.0
@@ -446,7 +495,14 @@ async def test_error_attribution(monkeypatch, tmp_path):
             return response
         finally:
             if trace is not None:
-                trace.record_judge(response)
+                trace.record_judge_call(
+                    name=self.reward_name,
+                    request={
+                        "model": self.config.model,
+                        "messages": [{"role": "user", "content": messages}],
+                    },
+                    response=response,
+                )
 
     monkeypatch.setattr(Judge, "complete", gibberish_judge)
     taskset = JudgedTaskset(
@@ -463,7 +519,7 @@ async def test_error_attribution(monkeypatch, tmp_path):
             trace, runtime=None
         )
     assert trace.rewards["reference"] is None  # seeded: expected but never scored
-    assert len(trace.info["judge"]) == 1  # the billed call is still recorded
+    assert len(trace.info["judge_calls"]) == 1  # the billed call is still recorded
 
 
 # --- rubric --------------------------------------------------------------------------------
@@ -542,7 +598,7 @@ async def test_rubric_score(tmp_path, fake_judge_model):
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 0.75
     assert trace.metrics == {"rubric/mentions_paris": 1.0, "rubric/is_polite": 0.0}
-    assert len(trace.info["judge"]) == 1  # one call for the whole rubric
+    assert len(trace.info["judge_calls"]) == 1  # one call for the whole rubric
 
 
 async def test_rubric_verdict_mismatch_raises(tmp_path, monkeypatch):
@@ -652,7 +708,7 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
         trace.rewards["quality"].score == 0.75
     )  # the rubric's aggregate, under its `name`
     assert (
-        len(trace.info["judge"]) == 2
+        len(trace.info["judge_calls"]) == 2
     )  # every judge call recorded (rubric = one call)
 
 

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/verifiers/v1/cli/replay.py

@@ -156,7 +156,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
             else:
                 st.state = "running"
                 # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge", None)
+                trace.info.pop("judge_calls", None)
                 trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/verifiers/v1/harnesses/prime_agent/harness.py

@@ -19,10 +19,10 @@ logger = logging.getLogger(__name__)
 GITHUB_RELEASE_URL = (
     "https://github.com/PrimeIntellect-ai/prime-agent/releases/download"
 )
-PRIME_AGENT_COMMIT: Literal["514633727bf26d74f39f3119c2b0e31a5ceb2a9d"] = (
-    "514633727bf26d74f39f3119c2b0e31a5ceb2a9d"
+PRIME_AGENT_COMMIT: Literal["81ae3cb34d27d38ee37f9e205a1e73694993b344"] = (
+    "81ae3cb34d27d38ee37f9e205a1e73694993b344"
 )
-PRIME_AGENT_VERSION = "0.8.1"
+PRIME_AGENT_VERSION = "0.9.1"
 PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent"
 STATE_ROOT = "/tmp/vf-prime-agent-runs"
 SKILLS_DIR = ".agents/skills"
@@ -51,10 +51,10 @@ for tarball in "$agent_tarball" "$ai_tarball" "$core_tarball" "$tui_tarball"; do
         "$release_url/$tarball" -o "$download_dir/$tarball"
 done
 printf '%s  %s\n' \
-    '46c24db1782dd31adc35d5c6cbcc75564faba6ced3bf2ccf03d836ee77134475' "$agent_tarball" \
-    'f6c3bdb6093bc24a327546fe865ef9a4a172c734fcd4c4093e30c19476f0134d' "$ai_tarball" \
-    '0cc3660953545f8ac9a7e704fcb9875f954d58c3085304080ef615c280aa5748' "$core_tarball" \
-    'bd07bccee0ca495565b1d62e9411f3fdebe49e3dfa52870564f08af5e61fde15' "$tui_tarball" \
+    '573bce0cd004fc62052e9a924089941b7f39266ab71e66a94c85a1f9d35835ba' "$agent_tarball" \
+    '11b5b4cf67b6bb2d3420a44fb69181bc9d94d81e69a2b4fde07eb9c99f5faf4f' "$ai_tarball" \
+    'fb6f3a5dcc8b69c5eeb3beff722b5e0f09885c14849db50bc1d7c0f1d064151c' "$core_tarball" \
+    '4f3eaca2814944d3993073e0b88c0bde54a641ddb5132fbe107e392b997e38ec' "$tui_tarball" \
     > "$download_dir/SHA256SUMS"
 (cd "$download_dir" && sha256sum -c SHA256SUMS)
 mkdir "$download_dir/package-root"
@@ -87,7 +87,7 @@ PRIME_AGENT_BOOTSTRAP_TOOLS_ON_INSTALL=1 npm install -g \
 
 
 class PrimeAgentHarnessConfig(HarnessConfig):
-    commit: Literal["514633727bf26d74f39f3119c2b0e31a5ceb2a9d"] = PRIME_AGENT_COMMIT
+    commit: Literal["81ae3cb34d27d38ee37f9e205a1e73694993b344"] = PRIME_AGENT_COMMIT
     """Prime Agent main commit to install."""
 
     autonomous: bool = False

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/verifiers/v1/harnesses/rlm/harness.py

@@ -5,7 +5,14 @@ import logging
 import shlex
 from typing import Any, Literal
 
-from pydantic import BaseModel, ConfigDict, Field, PositiveInt, model_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    NonNegativeInt,
+    PositiveInt,
+    model_validator,
+)
 from pydantic_config import BaseConfig
 
 from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn, JsonObject
@@ -17,7 +24,7 @@ from verifiers.v1.trace import Trace
 
 logger = logging.getLogger(__name__)
 
-BuiltinSkill = Literal["edit", "search"]
+BuiltinSkill = Literal["bash", "edit", "fetch", "search"]
 
 RLM_REPO = "github.com/PrimeIntellect-ai/nano-rlm.git"
 RLM_CACHE_DIR = "/tmp/vf-rlm"
@@ -40,18 +47,45 @@ class CompactionConfig(BaseConfig):
     summarize_at_tokens: PositiveInt | None = None
     """Compact at this token count. When unset, compact when 16k tokens remain below the
     model context window when the provider advertises it."""
+    max_compactions: PositiveInt | None = None
+    """Compactions per session before the engine stops compacting; `None` =
+    nano-rlm's default."""
 
 
 class RLMHarnessConfig(HarnessConfig):
-    version: str = Field(default="4ef3438", min_length=1)
-    """Git ref (branch, tag, or commit) of nano-rlm to install."""
-    max_depth: int = 0
-    """Recursion depth RLM may spawn sub-harnesses to."""
+    version: str = Field(default="dd2c04f", min_length=1)
+    """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every
+    field this harness puts on the wire, i.e. be at least the default ref."""
+    max_depth: NonNegativeInt | None = None
+    """Recursion depth RLM may spawn sub-agents to; `None` = nano-rlm's default (1).
+    Set 0 to disable recursion."""
     builtin_skills: list[BuiltinSkill] = Field(default_factory=list)
     """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none.
     The tool set is fixed (ipython); the base `skills` field takes SKILL.md paths."""
-    compaction: CompactionConfig | None = None
-    """Context compaction policy. Set an empty config to use automatic thresholds."""
+    compaction: CompactionConfig | bool | None = None
+    """Context compaction: a `[compaction]` section (or `true`) enables it with the
+    given settings, `false` disables it (an overflowing session then fails), and
+    unset defers to nano-rlm's default (on, automatic thresholds, bounded by its
+    default 1M tree-token budget)."""
+    max_concurrent_subagents: PositiveInt | None = None
+    """Sub-agents running at once per session tree; `None` = nano-rlm's default (4),
+    raised to an explicit `max_depth` when needed to keep the policy valid."""
+    max_total_turns: PositiveInt | None = None
+    """Tree-total turn budget (one turn = one work-loop model call, any engine); every
+    engine stops before its next call once spent. `None` = uncapped."""
+    max_total_tokens: PositiveInt | None = None
+    """Tree-total budget of NEW tokens (completion + uncached prompt) across the session
+    tree; once spent every engine stops and no further sub-agents spawn. `None` = unbounded."""
+    max_tool_output_bytes: PositiveInt | None = None
+    """Byte budget for a single tool result entering the conversation (middle truncation);
+    overrides rlm's built-in 20KB default in either direction."""
+    append_to_system_prompt: str | None = None
+    """Appended to the root engine's system prompt, after the taskset's system prompt."""
+    subagent_append_to_system_prompt: str | None = None
+    """Append for sub-agent engines that can still delegate (depth >= 1, below
+    `max_depth`); unset falls back to `append_to_system_prompt`."""
+    leaf_append_to_system_prompt: str | None = None
+    """Append for depth-capped leaf engines; unset falls back to the sub-agent append."""
 
     @model_validator(mode="after")
     def reject_disabled_tools(self) -> "RLMHarnessConfig":
@@ -60,6 +94,15 @@ class RLMHarnessConfig(HarnessConfig):
                 "the rlm harness has a fixed tool set (ipython) and does not support "
                 "`disabled_tools`; use `builtin_skills` to enable built-in skills instead."
             )
+        if (
+            self.max_depth is not None
+            and self.max_concurrent_subagents is not None
+            and self.max_concurrent_subagents < self.max_depth
+        ):
+            raise ValueError(
+                "`max_concurrent_subagents` must be at least `max_depth` "
+                "(nano-rlm rejects the policy otherwise)."
+            )
         return self
 
 
@@ -109,6 +152,27 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]):
         system_prompt: str | None,
     ) -> JsonObject:
         compaction = self.config.compaction
+        max_concurrent_subagents = self.config.max_concurrent_subagents
+        if max_concurrent_subagents is None and self.config.max_depth is not None:
+            max_concurrent_subagents = max(4, self.config.max_depth)
+        policy_knobs: dict[str, Any] = {
+            "max_depth": self.config.max_depth,
+            "max_concurrent_subagents": max_concurrent_subagents,
+            "max_total_turns": self.config.max_total_turns,
+            "max_total_tokens": self.config.max_total_tokens,
+            "max_tool_output_bytes": self.config.max_tool_output_bytes,
+        }
+        if isinstance(compaction, bool):
+            policy_knobs["compaction"] = compaction
+        elif compaction is not None:
+            policy_knobs["compaction"] = True
+            policy_knobs["summarize_at_tokens"] = compaction.summarize_at_tokens
+            policy_knobs["max_compactions"] = compaction.max_compactions
+        appends = [
+            text
+            for text in (system_prompt, self.config.append_to_system_prompt)
+            if text
+        ]
         payload = {
             "session_id": trace.id,
             "model": ctx.model,
@@ -116,16 +180,17 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]):
                 "base_url": endpoint,
                 "api_key": secret,
             },
+            # None = passthrough: the key stays off the wire and nano-rlm's own
+            # default applies.
             "policy": {
-                "max_depth": self.config.max_depth,
-                "compaction": compaction is not None,
-                "summarize_at_tokens": (
-                    compaction.summarize_at_tokens if compaction else None
-                ),
-                "max_concurrent_subagents": max(4, self.config.max_depth),
+                key: value for key, value in policy_knobs.items() if value is not None
             },
             "system_prompt_path": None,
-            "append_to_system_prompt": system_prompt,
+            "append_to_system_prompt": "\n\n".join(appends) or None,
+            "subagent_append_to_system_prompt": (
+                self.config.subagent_append_to_system_prompt
+            ),
+            "leaf_append_to_system_prompt": self.config.leaf_append_to_system_prompt,
             "skills": list(self.config.builtin_skills),
             "kernel_env": runtime.env,
             "search_api_key": self.config.resolved_env.get("SERPER_API_KEY"),

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/verifiers/v1/judge.py

@@ -32,7 +32,8 @@ in which case `JudgeResponse.parsed` is the validated pydantic object.
 A judge is cheap to construct (the HTTP client is opened per call, inside `complete`, and
 closed when the call returns), so build it where you use it.
 
-Passing `trace=` records the call onto it — a typed record appended to `trace.info["judge"]` and
+Passing `trace=` records the call onto it — a request/response record appended to
+`trace.info["judge_calls"]` and
 the call's tokens + cost added to `trace.extra_usage` (kept separate from the agent's `trace.usage`),
 so judge behaviour and spend are no longer invisible. The record lands even if the judge refuses, an
 empty structured output comes back, or `parse` raises (the request was already billed). Omit `trace`
@@ -210,7 +211,14 @@ class Judge(Generic[ParsedT, ConfigT]):
             return response
         finally:
             if trace is not None and response is not None:
-                trace.record_judge(response)
+                trace.record_judge_call(
+                    name=self.reward_name,
+                    request={
+                        "model": kwargs["model"],
+                        "messages": kwargs["messages"],
+                    },
+                    response=response,
+                )
 
     async def evaluate(
         self, *, trace: Trace | None = None, **fields: Any

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/verifiers/v1/judges/rubric.py

@@ -24,7 +24,7 @@ RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text(
 
 # Appended to the prompt when `structured_output` is off, so the reply is JSON we can parse. Each
 # criterion carries a one-sentence `reason` written *before* the verdict (chain-of-thought), so the
-# verdict follows from it and the reasoning is auditable in `trace.info["judge"]`.
+# verdict follows from it and the reasoning is auditable in `trace.info["judge_calls"]`.
 JSON_SUFFIX = (
     "\n\nRespond with ONLY a JSON object and nothing else, in exactly this shape:\n"
     '{"verdicts": [{"name": "<criterion name>", "reason": "<one sentence citing specific '

{verifiers-0.3.2.dev36 → verifiers-0.3.2.dev41}/verifiers/v1/trace.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import time
 import traceback
 import uuid
@@ -672,8 +673,26 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         reward = Reward(score=float(value), weight=float(weight))
         self.rewards[name] = reward
 
-    def record_judge(self, response: JudgeResponse) -> None:
-        self.info.setdefault("judge", []).append(response.model_dump())
+    def record_judge_call(
+        self,
+        *,
+        name: str,
+        request: Mapping[str, Any],
+        response: JudgeResponse,
+    ) -> None:
+        """Record one complete judge request/response exchange."""
+        response_record = response.model_dump()
+        message = AssistantMessage(content=response_record.pop("text"))
+        self.info.setdefault("judge_calls", []).append(
+            {
+                "name": name,
+                "request": copy.deepcopy(dict(request)),
+                "response": {
+                    "message": message.model_dump(),
+                    **response_record,
+                },
+            }
+        )
         if response.usage is not None:
             self.extra_usage.append(response.usage)