verifiers 0.4.1.dev63__tar.gz → 0.4.1.dev65__tar.gz

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev63
+Version: 0.4.1.dev65
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -40,7 +40,7 @@ Requires-Dist: prime-sandboxes>=0.5.0
 Requires-Dist: prime-tunnel>=0.1.14
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
-Requires-Dist: renderers[multimodal]>=0.1.12
+Requires-Dist: renderers[multimodal]>=0.1.13.dev16
 Requires-Dist: rich>=11.0.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
     "prime-pydantic-config[toml]>=0.4.3",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
-    "renderers[multimodal]>=0.1.12",
+    "renderers[multimodal]>=0.1.13.dev16",
     "typing_extensions>=4.12.2",
     "mcp==2.0.0",
 ]

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/tests/integration/fixtures/counter_tool.py

@@ -21,6 +21,8 @@ class CounterToolset(vf.Toolset[vf.ToolsetConfig, CounterState]):
 
 class CounterTaskConfig(vf.TaskConfig):
     tools: vf.ToolsetConfig = vf.ToolsetConfig()
+    constant_reward: bool = False
+    """Score every rollout 1, so a skipped score (reward 0) stands out."""
 
 
 class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
@@ -30,7 +32,7 @@ class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
 
     @vf.reward(weight=1.0)
     async def counted(self, trace: vf.Trace) -> float:
-        return float(trace.state.count >= 2)
+        return float(self.config.constant_reward or trace.state.count >= 2)
 
 
 class CounterConfig(vf.TasksetConfig):

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/tests/integration/test_limits.py

@@ -41,6 +41,44 @@ async def test_rollout_timeout(run):
     assert trace.stop_condition == "agent_timeout"
 
 
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_truncated(run, skip):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        taskset_overrides={"task": {"constant_reward": True}},
+        env={"agent": {"max_turns": 1, "scoring": {"skip_on_truncated": skip}}},
+    )
+    (trace,) = episode.traces
+    assert trace.is_truncated and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "truncated"
+    else:
+        assert trace.reward == 1
+        assert "scoring_skipped" not in trace.info
+
+
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_timeout(run, skip):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        rollout_timeout=1,
+        taskset_overrides={"task": {"constant_reward": True}},
+        env={"agent": {"scoring": {"skip_on_timeout": skip}}},
+    )
+    (trace,) = episode.traces
+    assert trace.is_timeout and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "timeout"
+    else:
+        assert trace.reward == 1
+
+
 @pytest.mark.parametrize("harness,harness_runtime", grid(needs=("mcp",)), indirect=True)
 async def test_max_turns(run, harness, harness_runtime):
     (episode,) = await run(

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/tests/unit/test_trace.py

@@ -9,6 +9,7 @@ from verifiers.v1.configs.agent import resolve_agent
 from verifiers.v1.dialects.chat import ChatDialect
 from verifiers.v1.dialects.responses import ResponsesDialect, fold_assistant
 from verifiers.v1.graph import MessageNode, prepare_turn
+from verifiers.v1.session import RolloutLimits
 from verifiers.v1.types import AssistantMessage, UserMessage
 from verifiers.v1.utils.trace_store import write_episode
 
@@ -266,3 +267,29 @@ def test_sampling_mask_from_packed_matches_lists():
     ):
         np.testing.assert_array_equal(mask.ids, ids)
         np.testing.assert_array_equal(mask.counts, counts)
+
+
+def test_max_turn_tokens_limit():
+    limits = RolloutLimits(max_turn_tokens=16)
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt=None)),
+    )
+    trace.calls.append(vf.ModelCall(finish_reason="stop"))
+    assert limits.reached(trace) is None
+    # A length finish below the cap ran out of context: not this limit.
+    usage = vf.Usage(prompt_tokens=100, completion_tokens=8)
+    trace.calls.append(vf.ModelCall(finish_reason="length", usage=usage))
+    assert limits.reached(trace) is None
+    usage = vf.Usage(prompt_tokens=100, completion_tokens=16)
+    trace.calls.append(vf.ModelCall(finish_reason="length", usage=usage))
+    assert limits.reached(trace) == "max_turn_tokens"
+    trace.stop("max_turn_tokens")
+    assert trace.is_truncated
+
+
+@pytest.mark.parametrize("max_tokens,expected", [(None, 16), (8, 8), (64, 16)])
+def test_max_turn_tokens_caps_sampling(max_tokens, expected):
+    spec = vf.AgentConfig(max_turn_tokens=16)
+    sampling = vf.SamplingConfig(max_tokens=max_tokens)
+    assert resolve_agent(spec, sampling=sampling).sampling.max_tokens == expected

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/verifiers/v1/agent.py

@@ -575,6 +575,7 @@ class Agent:
             max_input_tokens=config.max_input_tokens,
             max_output_tokens=config.max_output_tokens,
             max_total_tokens=config.max_total_tokens,
+            max_turn_tokens=config.max_turn_tokens,
         )
         self.timeout = config.timeout
         # Env episode agents replace this with the episode's agent semaphore

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/verifiers/v1/configs/agent.py

@@ -66,6 +66,11 @@ class AgentConfig(BaseConfig):
     """Max output tokens per run (None = no limit)."""
     max_total_tokens: int | None = None
     """Max total tokens per run (None = no limit)."""
+    max_turn_tokens: int | None = Field(None, gt=0)
+    """Max output tokens per model response (None = no limit). Caps the sampling
+    `max_tokens` the framework sets on every request, whatever the harness asks for,
+    and stops the run once a response hits it. `sampling.max_tokens` alone caps each
+    response without stopping the run."""
 
     timeout: TimeoutConfig = TimeoutConfig()
     retries: RetryConfig = RetryConfig()
@@ -147,6 +152,11 @@ def resolve_agent(
     defaults = sampling or SamplingConfig()
     overrides = (spec.sampling or SamplingConfig()).model_dump(exclude_unset=True)
     merged = {**defaults.model_dump(), **merge_defaults(defaults, overrides)}
+    if spec.max_turn_tokens is not None:
+        max_tokens = merged.get("max_tokens")
+        merged["max_tokens"] = min(
+            max_tokens or spec.max_turn_tokens, spec.max_turn_tokens
+        )
     return spec.model_copy(
         update={
             "harness": spec.harness if spec.harness is not None else harness,

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/verifiers/v1/configs/scoring.py

@@ -24,6 +24,12 @@ class ScoringConfig(BaseConfig):
     combined with the runtime's own rules the same way. None keeps the default: isolated
     scoring runs under `TaskData.network`, shared scoring keeps the agent phase's network,
     and Harbor's shared grading lifts the policy."""
+    skip_on_truncated: bool = False
+    """Skip task scoring for a truncated trace (`Trace.is_truncated`): it keeps no
+    rewards, so its reward is 0."""
+    skip_on_timeout: bool = False
+    """Skip task scoring for a trace whose stage deadline expired before scoring
+    (`Trace.is_timeout`): it keeps no rewards, so its reward is 0."""
 
     @field_serializer("runtime")
     def serialize_runtime(

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/verifiers/v1/rollout.py

@@ -721,6 +721,14 @@ class Rollout:
             "agent_completed",
         )
 
+    def _scoring_skip_reason(self) -> str | None:
+        """Why the resolved scoring policy skips task scoring for this trace, if it does."""
+        if self._scoring_policy.skip_on_truncated and self.trace.is_truncated:
+            return "truncated"
+        if self._scoring_policy.skip_on_timeout and self.trace.is_timeout:
+            return "timeout"
+        return None
+
     def _metrics_deadline(self) -> float:
         """Best-effort metrics get at most 30 seconds, and a tenth of a finite scoring budget."""
         budget = (
@@ -848,7 +856,14 @@ class Rollout:
                         boundary(TaskError, "scoring"),
                     ):
                         # Cross-trace judgement runs later, after the runtime is gone.
-                        if grader is not None:
+                        if skip_reason := self._scoring_skip_reason():
+                            logger.info(
+                                "rollout %s: scoring skipped (%s)",
+                                trace.id,
+                                skip_reason,
+                            )
+                            trace.info["scoring_skipped"] = skip_reason
+                        elif grader is not None:
                             try:
                                 await grade_task(
                                     grader,

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/verifiers/v1/session.py

@@ -87,12 +87,15 @@ class RolloutLimits:
     a @stop — and becomes the trace's stop condition. Each caps a trace computed property:
     `max_turns` -> num_turns, `max_input_tokens` -> num_input_tokens, `max_output_tokens` ->
     num_output_tokens, `max_total_tokens` -> num_total_tokens. Token caps are soft by one turn:
-    they're checked between turns, so the turn that crosses a cap still completes."""
+    they're checked between turns, so the turn that crosses a cap still completes.
+    `max_turn_tokens` caps each response (applied through sampling) and refuses the turn
+    after a response that hit it."""
 
     max_turns: int | None = None
     max_input_tokens: int | None = None
     max_output_tokens: int | None = None
     max_total_tokens: int | None = None
+    max_turn_tokens: int | None = None
 
     def reached(self, trace: Trace) -> str | None:
         """The name of the first limit `trace` has reached, or None if within all caps."""
@@ -113,6 +116,18 @@ class RolloutLimits:
             and trace.num_total_tokens >= self.max_total_tokens
         ):
             return "max_total_tokens"
+        if self.max_turn_tokens is not None:
+            last = next((c for c in reversed(trace.calls) if c.error is None), None)
+            # A length finish below the cap ran out of context, not turn tokens.
+            if (
+                last is not None
+                and last.finish_reason == "length"
+                and (
+                    last.usage is None
+                    or last.usage.completion_tokens >= self.max_turn_tokens
+                )
+            ):
+                return "max_turn_tokens"
         return None
 
 

{verifiers-0.4.1.dev63 → verifiers-0.4.1.dev65}/verifiers/v1/trace.py

@@ -648,6 +648,7 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
             "max_input_tokens",
             "max_output_tokens",
             "max_total_tokens",
+            "max_turn_tokens",
             "compaction_failed",
         ):
             return True