verifiers 0.4.1.dev64__tar.gz → 0.4.1.dev67__tar.gz

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev64
+Version: 0.4.1.dev67
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/tests/integration/fixtures/counter_tool.py

@@ -21,6 +21,8 @@ class CounterToolset(vf.Toolset[vf.ToolsetConfig, CounterState]):
 
 class CounterTaskConfig(vf.TaskConfig):
     tools: vf.ToolsetConfig = vf.ToolsetConfig()
+    constant_reward: bool = False
+    """Score every rollout 1, so a skipped score (reward 0) stands out."""
 
 
 class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
@@ -30,7 +32,12 @@ class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
 
     @vf.reward(weight=1.0)
     async def counted(self, trace: vf.Trace) -> float:
-        return float(trace.state.count >= 2)
+        return float(self.config.constant_reward or trace.state.count >= 2)
+
+
+def any_response(response: vf.Response) -> bool:
+    """A plugged stop that ends the run on its first model response."""
+    return True
 
 
 class CounterConfig(vf.TasksetConfig):

verifiers-0.4.1.dev67/tests/integration/test_limits.py

@@ -0,0 +1,116 @@
+"""The seat's caps stop a rollout that would run on: the counter task wants three tool
+turns (counter-tool), and each cap ends it after the first with its own stop
+condition. The turn cap also runs across the harness x runtime grid, since each program
+handles a refused request in its own way."""
+
+import pytest
+from grid import grid
+
+CAPS = [
+    pytest.param({"max_turns": 1}, "max_turns", id="max-turns"),
+    pytest.param({"max_input_tokens": 1}, "max_input_tokens", id="max-input-tokens"),
+    pytest.param({"max_output_tokens": 1}, "max_output_tokens", id="max-output-tokens"),
+    pytest.param({"max_total_tokens": 1}, "max_total_tokens", id="max-total-tokens"),
+]
+
+
+@pytest.mark.parametrize("cap,stop_condition", CAPS)
+async def test_cap(run, cap, stop_condition):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        env={"agent": cap},
+    )
+    (trace,) = episode.traces
+    assert trace.num_branches == 1
+    assert trace.stop_condition == stop_condition
+    assert trace.is_truncated
+    assert trace.num_turns == 1
+
+
+async def test_rollout_timeout(run):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        rollout_timeout=1,
+    )
+    (trace,) = episode.traces
+    assert trace.is_timeout
+    assert trace.stop_condition == "agent_timeout"
+
+
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_truncated(run, skip):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        taskset_overrides={"task": {"constant_reward": True}},
+        env={"agent": {"max_turns": 1, "scoring": {"skip_on_truncated": skip}}},
+    )
+    (trace,) = episode.traces
+    assert trace.is_truncated and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "truncated"
+    else:
+        assert trace.reward == 1
+        assert "scoring_skipped" not in trace.info
+
+
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_timeout(run, skip):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        rollout_timeout=1,
+        taskset_overrides={"task": {"constant_reward": True}},
+        env={"agent": {"scoring": {"skip_on_timeout": skip}}},
+    )
+    (trace,) = episode.traces
+    assert trace.is_timeout and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "timeout"
+    else:
+        assert trace.reward == 1
+
+
+@pytest.mark.parametrize("harness,harness_runtime", grid(needs=("mcp",)), indirect=True)
+async def test_max_turns(run, harness, harness_runtime):
+    (episode,) = await run(
+        "counter-tool",
+        harness=harness,
+        runtime=harness_runtime,
+        max_turns=1,
+        max_tokens=8192,
+    )
+    (trace,) = episode.traces
+    assert trace.num_branches == 1
+    assert trace.stop_condition == "max_turns", [e.message for e in trace.errors]
+    assert trace.num_turns == 1
+
+
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_stop(run, scripted_model, skip):
+    stop = {"fn": "counter_tool:any_response", "skip_scoring": skip}
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "subprocess"},
+        client=scripted_model,
+        taskset_overrides={
+            "task": {"constant_reward": True, "stops": {"any_response": stop}}
+        },
+    )
+    (trace,) = episode.traces
+    assert trace.stop_condition == "any_response" and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "any_response"
+    else:
+        assert trace.reward == 1
+        assert "scoring_skipped" not in trace.info

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/tests/unit/test_task.py

@@ -36,6 +36,10 @@ class HookTask(vf.Task[HookData]):
     async def single_turn(self, trace: vf.Trace) -> bool:
         return trace.num_turns >= 1
 
+    @vf.stop(skip_scoring=True)
+    async def no_reply(self, trace: vf.Trace) -> bool:
+        return trace.last_reply is None
+
     @vf.reward
     async def lcs(self, trace: vf.Trace) -> float:
         return 0.5
@@ -49,7 +53,7 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
     fns = tmp_path / "fns.py"
     fns.write_text(PLUGGED_FNS_PY)
     config = vf.TaskConfig(
-        stops={"single_turn": vf.DecoratedFunctionConfig(fn=f"{fns}:two_turns")},
+        stops={"single_turn": vf.StopFunctionConfig(fn=f"{fns}:two_turns")},
         metrics={"reply_length": vf.DecoratedFunctionConfig(fn=f"{fns}:reply_length")},
         rewards={
             "exact_match": vf.RewardFunctionConfig(fn=f"{fns}:exact_match", weight=0.5),
@@ -70,9 +74,10 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
     )
 
     # the plugged `single_turn` replaces the decorated one: no stop after one turn
-    (stop,) = task.hooks("stop")
+    no_reply, stop = task.hooks("stop")
     assert stop.__name__ == "single_turn"
     assert not await stop(trace)
+    assert no_reply.stop_skip_scoring and not stop.stop_skip_scoring
 
     await task.score(trace)
     # `lcs` scored by the plugged marker (config wins the name clash), config weight
@@ -102,3 +107,27 @@ async def test_defer_scoring_defers_only_task_scoring() -> None:
     assert trace.rewards == {}
     await task.score(trace)
     assert set(trace.rewards) == {"fmt", "lcs"}
+
+
+def test_plugged_stops_set_skip_scoring(tmp_path) -> None:
+    fns = tmp_path / "fns.py"
+    fns.write_text(PLUGGED_FNS_PY)
+    data = HookData(idx=0, prompt="abc")
+
+    def skips(stops: dict) -> dict[str, bool]:
+        config = vf.TaskConfig(stops=stops)
+        return {
+            fn.__name__: fn.stop_skip_scoring
+            for fn in HookTask(data, config).hooks("stop")
+        }
+
+    # a fresh plugged stop scores by default; a fn-less entry keeps or overrides the decorator's
+    assert skips({"two": vf.StopFunctionConfig(fn=f"{fns}:two_turns")})["two"] is False
+    assert skips(
+        {"two": vf.StopFunctionConfig(fn=f"{fns}:two_turns", skip_scoring=True)}
+    )["two"]
+    assert skips({"no_reply": vf.StopFunctionConfig()})["no_reply"] is True
+    assert (
+        skips({"no_reply": vf.StopFunctionConfig(skip_scoring=False)})["no_reply"]
+        is False
+    )

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/tests/unit/test_trace.py

@@ -9,6 +9,7 @@ from verifiers.v1.configs.agent import resolve_agent
 from verifiers.v1.dialects.chat import ChatDialect
 from verifiers.v1.dialects.responses import ResponsesDialect, fold_assistant
 from verifiers.v1.graph import MessageNode, prepare_turn
+from verifiers.v1.session import RolloutLimits
 from verifiers.v1.types import AssistantMessage, UserMessage
 from verifiers.v1.utils.trace_store import write_episode
 
@@ -266,3 +267,29 @@ def test_sampling_mask_from_packed_matches_lists():
     ):
         np.testing.assert_array_equal(mask.ids, ids)
         np.testing.assert_array_equal(mask.counts, counts)
+
+
+def test_max_turn_tokens_limit():
+    limits = RolloutLimits(max_turn_tokens=16)
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt=None)),
+    )
+    trace.calls.append(vf.ModelCall(finish_reason="stop"))
+    assert limits.reached(trace) is None
+    # A length finish below the cap ran out of context: not this limit.
+    usage = vf.Usage(prompt_tokens=100, completion_tokens=8)
+    trace.calls.append(vf.ModelCall(finish_reason="length", usage=usage))
+    assert limits.reached(trace) is None
+    usage = vf.Usage(prompt_tokens=100, completion_tokens=16)
+    trace.calls.append(vf.ModelCall(finish_reason="length", usage=usage))
+    assert limits.reached(trace) == "max_turn_tokens"
+    trace.stop("max_turn_tokens")
+    assert trace.is_truncated
+
+
+@pytest.mark.parametrize("max_tokens,expected", [(None, 16), (8, 8), (64, 16)])
+def test_max_turn_tokens_caps_sampling(max_tokens, expected):
+    spec = vf.AgentConfig(max_turn_tokens=16)
+    sampling = vf.SamplingConfig(max_tokens=max_tokens)
+    assert resolve_agent(spec, sampling=sampling).sampling.max_tokens == expected

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/__init__.py

@@ -56,6 +56,7 @@ if TYPE_CHECKING:
     from verifiers.v1.configs.task import (
         DecoratedFunctionConfig,
         RewardFunctionConfig,
+        StopFunctionConfig,
         TaskConfig,
     )
     from verifiers.v1.configs.taskset import SharedTasksetConfig, TasksetConfig
@@ -301,6 +302,7 @@ _NAME_TO_MODULE = {
     "State": "verifiers.v1.state",
     "StateT": "verifiers.v1.state",
     "StaticPoolConfig": "verifiers.v1.configs.serve",
+    "StopFunctionConfig": "verifiers.v1.configs.task",
     "SubprocessConfig": "verifiers.v1.configs.runtime",
     "SystemMessage": "verifiers.v1.types",
     "TRACE_VERSION": "verifiers.v1.trace",
@@ -480,6 +482,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "SharedTasksetConfig",
     "DecoratedFunctionConfig",
     "RewardFunctionConfig",
+    "StopFunctionConfig",
     "BaseConfig",
     "Harness",
     "HarnessSession",

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/agent.py

@@ -575,6 +575,7 @@ class Agent:
             max_input_tokens=config.max_input_tokens,
             max_output_tokens=config.max_output_tokens,
             max_total_tokens=config.max_total_tokens,
+            max_turn_tokens=config.max_turn_tokens,
         )
         self.timeout = config.timeout
         # Env episode agents replace this with the episode's agent semaphore

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/configs/agent.py

@@ -66,6 +66,11 @@ class AgentConfig(BaseConfig):
     """Max output tokens per run (None = no limit)."""
     max_total_tokens: int | None = None
     """Max total tokens per run (None = no limit)."""
+    max_turn_tokens: int | None = Field(None, gt=0)
+    """Max output tokens per model response (None = no limit). Caps the sampling
+    `max_tokens` the framework sets on every request, whatever the harness asks for,
+    and stops the run once a response hits it. `sampling.max_tokens` alone caps each
+    response without stopping the run."""
 
     timeout: TimeoutConfig = TimeoutConfig()
     retries: RetryConfig = RetryConfig()
@@ -147,6 +152,11 @@ def resolve_agent(
     defaults = sampling or SamplingConfig()
     overrides = (spec.sampling or SamplingConfig()).model_dump(exclude_unset=True)
     merged = {**defaults.model_dump(), **merge_defaults(defaults, overrides)}
+    if spec.max_turn_tokens is not None:
+        max_tokens = merged.get("max_tokens")
+        merged["max_tokens"] = min(
+            max_tokens or spec.max_turn_tokens, spec.max_turn_tokens
+        )
     return spec.model_copy(
         update={
             "harness": spec.harness if spec.harness is not None else harness,

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/configs/scoring.py

@@ -24,6 +24,12 @@ class ScoringConfig(BaseConfig):
     combined with the runtime's own rules the same way. None keeps the default: isolated
     scoring runs under `TaskData.network`, shared scoring keeps the agent phase's network,
     and Harbor's shared grading lifts the policy."""
+    skip_on_truncated: bool = False
+    """Skip task scoring for a truncated trace (`Trace.is_truncated`): it keeps no
+    rewards, so its reward is 0."""
+    skip_on_timeout: bool = False
+    """Skip task scoring for a trace whose stage deadline expired before scoring
+    (`Trace.is_timeout`): it keeps no rewards, so its reward is 0."""
 
     @field_serializer("runtime")
     def serialize_runtime(

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/configs/task.py

@@ -14,13 +14,21 @@ class DecoratedFunctionConfig(BaseConfig):
     fn: str | None = None
     """Import path to the function: `pkg.module.function`, `pkg.module:function`,
     or `path/to/file.py:function`. Unset keeps the task's decorated method with the
-    entry's name and only overrides its metadata (priority, weight)."""
+    entry's name and only overrides its metadata (priority, weight, skip_scoring)."""
 
     priority: int | None = None
     """Execution order, like the decorator's — higher runs first, ties break by name.
     Unset keeps the replaced method's priority (0 for a fresh function)."""
 
 
+class StopFunctionConfig(DecoratedFunctionConfig):
+    """A plugged function standing in for a `@vf.stop` task method."""
+
+    skip_scoring: bool | None = None
+    """Skip task scoring for a trace this stop ends: it keeps no rewards, so its reward
+    is 0. Unset keeps the replaced method's setting (false for a fresh function)."""
+
+
 class RewardFunctionConfig(DecoratedFunctionConfig):
     """A plugged function standing in for a `@vf.reward` task method."""
 
@@ -39,7 +47,7 @@ class TaskConfig(BaseConfig):
     judges: Judges = Field(default_factory=list)
     """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
 
-    stops: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
+    stops: dict[str, StopFunctionConfig] = Field(default_factory=dict)
     """Typed `Request`, `Response`, or `Trace` stop predicates plugged by name and
     merged with the task's `@vf.stop` methods (a plugged function replaces a
     decorated one with the same name)."""

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/rollout.py

@@ -157,6 +157,9 @@ class Rollout:
             for fn in discover_decorated(task, "intercept")
         ]
         stops = [(hook_boundary(fn, allow_trace=True), fn) for fn in task.hooks("stop")]
+        self._skip_scoring_stops = {
+            fn.__name__ for _, fn in stops if getattr(fn, "stop_skip_scoring", False)
+        }
         execution_policy = runtime_config
         self._session = RolloutSession(
             ctx=ctx,
@@ -721,6 +724,17 @@ class Rollout:
             "agent_completed",
         )
 
+    def _scoring_skip_reason(self) -> str | None:
+        """Why the resolved scoring policy, or the `skip_scoring` stop that ended this trace,
+        skips task scoring for it, if either does."""
+        if self._scoring_policy.skip_on_truncated and self.trace.is_truncated:
+            return "truncated"
+        if self._scoring_policy.skip_on_timeout and self.trace.is_timeout:
+            return "timeout"
+        if self.trace.stop_condition in self._skip_scoring_stops:
+            return self.trace.stop_condition
+        return None
+
     def _metrics_deadline(self) -> float:
         """Best-effort metrics get at most 30 seconds, and a tenth of a finite scoring budget."""
         budget = (
@@ -848,7 +862,14 @@ class Rollout:
                         boundary(TaskError, "scoring"),
                     ):
                         # Cross-trace judgement runs later, after the runtime is gone.
-                        if grader is not None:
+                        if skip_reason := self._scoring_skip_reason():
+                            logger.info(
+                                "rollout %s: scoring skipped (%s)",
+                                trace.id,
+                                skip_reason,
+                            )
+                            trace.info["scoring_skipped"] = skip_reason
+                        elif grader is not None:
                             try:
                                 await grade_task(
                                     grader,

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/session.py

@@ -87,12 +87,15 @@ class RolloutLimits:
     a @stop — and becomes the trace's stop condition. Each caps a trace computed property:
     `max_turns` -> num_turns, `max_input_tokens` -> num_input_tokens, `max_output_tokens` ->
     num_output_tokens, `max_total_tokens` -> num_total_tokens. Token caps are soft by one turn:
-    they're checked between turns, so the turn that crosses a cap still completes."""
+    they're checked between turns, so the turn that crosses a cap still completes.
+    `max_turn_tokens` caps each response (applied through sampling) and refuses the turn
+    after a response that hit it."""
 
     max_turns: int | None = None
     max_input_tokens: int | None = None
     max_output_tokens: int | None = None
     max_total_tokens: int | None = None
+    max_turn_tokens: int | None = None
 
     def reached(self, trace: Trace) -> str | None:
         """The name of the first limit `trace` has reached, or None if within all caps."""
@@ -113,6 +116,18 @@ class RolloutLimits:
             and trace.num_total_tokens >= self.max_total_tokens
         ):
             return "max_total_tokens"
+        if self.max_turn_tokens is not None:
+            last = next((c for c in reversed(trace.calls) if c.error is None), None)
+            # A length finish below the cap ran out of context, not turn tokens.
+            if (
+                last is not None
+                and last.finish_reason == "length"
+                and (
+                    last.usage is None
+                    or last.usage.completion_tokens >= self.max_turn_tokens
+                )
+            ):
+                return "max_turn_tokens"
         return None
 
 

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/trace.py

@@ -648,6 +648,7 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
             "max_input_tokens",
             "max_output_tokens",
             "max_total_tokens",
+            "max_turn_tokens",
             "compaction_failed",
         ):
             return True

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/utils/decorators.py

@@ -83,12 +83,18 @@ def tool(func: F | None = None, name: str | None = None) -> F | Callable[[F], F]
 
 
 @overload
-def stop(func: F, priority: int = 0) -> F: ...
+def stop(func: F, priority: int = 0, skip_scoring: bool = False) -> F: ...
 @overload
-def stop(func: None = None, priority: int = 0) -> Callable[[F], F]: ...
-def stop(func: F | None = None, priority: int = 0) -> F | Callable[[F], F]:
-    """Stop when a typed `Request`, `Response`, or `Trace` predicate returns true."""
-    decorator = mark("stop", stop_priority=priority)
+def stop(
+    func: None = None, priority: int = 0, skip_scoring: bool = False
+) -> Callable[[F], F]: ...
+def stop(
+    func: F | None = None, priority: int = 0, skip_scoring: bool = False
+) -> F | Callable[[F], F]:
+    """Stop when a typed `Request`, `Response`, or `Trace` predicate returns true. With
+    `skip_scoring`, a trace this stop ends skips task scoring: it keeps no rewards, so
+    its reward is 0."""
+    decorator = mark("stop", stop_priority=priority, stop_skip_scoring=skip_scoring)
     return decorator if func is None else decorator(func)
 
 

{verifiers-0.4.1.dev64 → verifiers-0.4.1.dev67}/verifiers/v1/utils/loaders.py

@@ -15,7 +15,11 @@ from pydantic_config import BaseConfig
 from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig
-from verifiers.v1.configs.task import DecoratedFunctionConfig, RewardFunctionConfig
+from verifiers.v1.configs.task import (
+    DecoratedFunctionConfig,
+    RewardFunctionConfig,
+    StopFunctionConfig,
+)
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv
@@ -279,6 +283,11 @@ def load_plugged_fn(
         if weight is None:
             weight = getattr(fn, "_vf_weight", 1.0)
         plugged._vf_weight = weight
+    if isinstance(config, StopFunctionConfig):
+        skip_scoring = config.skip_scoring
+        if skip_scoring is None:
+            skip_scoring = getattr(fn, "stop_skip_scoring", False)
+        plugged.stop_skip_scoring = skip_scoring
     return plugged
 
 

verifiers-0.4.1.dev64/tests/integration/test_limits.py

@@ -1,56 +0,0 @@
-"""The seat's caps stop a rollout that would run on: the counter task wants three tool
-turns (counter-tool), and each cap ends it after the first with its own stop
-condition. The turn cap also runs across the harness x runtime grid, since each program
-handles a refused request in its own way."""
-
-import pytest
-from grid import grid
-
-CAPS = [
-    pytest.param({"max_turns": 1}, "max_turns", id="max-turns"),
-    pytest.param({"max_input_tokens": 1}, "max_input_tokens", id="max-input-tokens"),
-    pytest.param({"max_output_tokens": 1}, "max_output_tokens", id="max-output-tokens"),
-    pytest.param({"max_total_tokens": 1}, "max_total_tokens", id="max-total-tokens"),
-]
-
-
-@pytest.mark.parametrize("cap,stop_condition", CAPS)
-async def test_cap(run, cap, stop_condition):
-    (episode,) = await run(
-        "counter-tool",
-        harness="null",
-        runtime={"type": "prime"},
-        env={"agent": cap},
-    )
-    (trace,) = episode.traces
-    assert trace.num_branches == 1
-    assert trace.stop_condition == stop_condition
-    assert trace.is_truncated
-    assert trace.num_turns == 1
-
-
-async def test_rollout_timeout(run):
-    (episode,) = await run(
-        "counter-tool",
-        harness="null",
-        runtime={"type": "prime"},
-        rollout_timeout=1,
-    )
-    (trace,) = episode.traces
-    assert trace.is_timeout
-    assert trace.stop_condition == "agent_timeout"
-
-
-@pytest.mark.parametrize("harness,harness_runtime", grid(needs=("mcp",)), indirect=True)
-async def test_max_turns(run, harness, harness_runtime):
-    (episode,) = await run(
-        "counter-tool",
-        harness=harness,
-        runtime=harness_runtime,
-        max_turns=1,
-        max_tokens=8192,
-    )
-    (trace,) = episode.traces
-    assert trace.num_branches == 1
-    assert trace.stop_condition == "max_turns", [e.message for e in trace.errors]
-    assert trace.num_turns == 1