verifiers 0.1.15.dev157__tar.gz → 0.1.15.dev159__tar.gz

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev159}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev157
+Version: 0.1.15.dev159
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -54,17 +54,17 @@ Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.0.0; extra == 'browser'
 Provides-Extra: harnesses
-Requires-Dist: harnesses>=0.1.1; extra == 'harnesses'
+Requires-Dist: harnesses>=0.1.2; extra == 'harnesses'
 Provides-Extra: nemogym
-Requires-Dist: harnesses[nemogym]>=0.1.1; extra == 'nemogym'
-Requires-Dist: tasksets[nemogym]>=0.1.1; extra == 'nemogym'
+Requires-Dist: harnesses[nemogym]>=0.1.2; extra == 'nemogym'
+Requires-Dist: tasksets[nemogym]>=0.1.5; extra == 'nemogym'
 Provides-Extra: openenv
-Requires-Dist: tasksets[openenv]>=0.1.1; extra == 'openenv'
+Requires-Dist: tasksets[openenv]>=0.1.5; extra == 'openenv'
 Provides-Extra: openreward
-Requires-Dist: tasksets[openreward]>=0.1.1; extra == 'openreward'
+Requires-Dist: tasksets[openreward]>=0.1.5; extra == 'openreward'
 Provides-Extra: packages
-Requires-Dist: harnesses>=0.1.1; extra == 'packages'
-Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.1; extra == 'packages'
+Requires-Dist: harnesses>=0.1.2; extra == 'packages'
+Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.5; extra == 'packages'
 Provides-Extra: renderers
 Requires-Dist: renderers>=0.1.8.dev28; extra == 'renderers'
 Provides-Extra: rg
@@ -84,7 +84,7 @@ Provides-Extra: ta
 Requires-Dist: nltk; extra == 'ta'
 Requires-Dist: textarena; extra == 'ta'
 Provides-Extra: tasksets
-Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.1; extra == 'tasksets'
+Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.5; extra == 'tasksets'
 Description-Content-Type: text/markdown
 
 <p align="center">

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev159}/pyproject.toml

@@ -89,14 +89,14 @@ ta = [
     "textarena",
 ]
 tasksets = [
-    "tasksets[openenv,openreward,ta]>=0.1.1",
+    "tasksets[openenv,openreward,ta]>=0.1.5",
 ]
 harnesses = [
-    "harnesses>=0.1.1",
+    "harnesses>=0.1.2",
 ]
 packages = [
-    "harnesses>=0.1.1",
-    "tasksets[openenv,openreward,ta]>=0.1.1",
+    "harnesses>=0.1.2",
+    "tasksets[openenv,openreward,ta]>=0.1.5",
 ]
 browser = [
     "stagehand>=3.0.0",
@@ -104,14 +104,14 @@ browser = [
     "python-dotenv>=1.0.0",
 ]
 openenv = [
-    "tasksets[openenv]>=0.1.1",
+    "tasksets[openenv]>=0.1.5",
 ]
 openreward = [
-    "tasksets[openreward]>=0.1.1",
+    "tasksets[openreward]>=0.1.5",
 ]
 nemogym = [
-    "harnesses[nemogym]>=0.1.1",
-    "tasksets[nemogym]>=0.1.1",
+    "harnesses[nemogym]>=0.1.2",
+    "tasksets[nemogym]>=0.1.5",
 ]
 renderers = [
     "renderers>=0.1.8.dev28",

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev159}/tests/test_imports.py

@@ -17,6 +17,9 @@ PACKAGE_SYMBOLS = {
     "OpenCodeConfig",
     "Pi",
     "PiConfig",
+    "ReplayHarness",
+    "ReplayTaskset",
+    "ReplayTasksetConfig",
     "RLM",
     "RLMConfig",
     "Terminus2",

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev159}/tests/test_init_script.py

@@ -106,7 +106,7 @@ def test_init_openenv_writes_v1_taskset_template(tmp_path: Path) -> None:
     assert "taskset=vf.load_taskset(config=config.taskset)" in content
     assert "harness=vf.load_harness(config=config.harness)" in content
     assert "vf.OpenEnvEnv" not in content
-    assert '"tasksets[openenv]>=0.1.1"' in pyproject
+    assert '"tasksets[openenv]>=0.1.5"' in pyproject
 
 
 def test_init_openenv_multifile_exports_taskset_loader(tmp_path: Path) -> None:

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev159}/tests/test_v1_config_extension.py

@@ -2738,6 +2738,68 @@ def test_load_environment_validates_typed_env_config_arg(
     }
 
 
+def test_load_environment_accepts_top_level_taskset_harness_shorthand(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "typed_env_config_shorthand"
+    module = types.ModuleType(module_name)
+    seen: dict[str, object] = {}
+
+    def load_environment(split: str = "train", *, config: EnvConfig) -> Env:
+        seen["split"] = split
+        seen["config"] = config
+        return Env(
+            taskset=make_taskset(config=config.taskset),
+            harness=make_harness(config=config.harness),
+        )
+
+    module.load_environment = load_environment
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    env = vf.load_environment(
+        "typed-env-config-shorthand",
+        split="test",
+        taskset={"taskset_id": "top-level"},
+        harness={"model": {"name": "top-level-model"}},
+    )
+
+    assert seen["split"] == "test"
+    config = seen["config"]
+    assert isinstance(config, EnvConfig)
+    assert config.taskset.taskset_id == "top-level"
+    assert config.harness.model.name == "top-level-model"
+    assert env.taskset.config.taskset_id == "top-level"
+    assert env.harness.config.model.name == "top-level-model"
+    assert env.env_args == {
+        "split": "test",
+        "taskset": {"taskset_id": "top-level"},
+        "harness": {"model": {"name": "top-level-model"}},
+    }
+
+
+def test_load_environment_rejects_duplicate_top_level_config_shorthand(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "typed_env_config_duplicate_shorthand"
+    module = types.ModuleType(module_name)
+
+    def load_environment(config: EnvConfig) -> Env:
+        return Env(
+            taskset=make_taskset(config=config.taskset),
+            harness=make_harness(config=config.harness),
+        )
+
+    module.load_environment = load_environment
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    with pytest.raises(RuntimeError, match="not both"):
+        vf.load_environment(
+            "typed-env-config-duplicate-shorthand",
+            config={"taskset": {"taskset_id": "nested"}},
+            taskset={"taskset_id": "top-level"},
+        )
+
+
 def test_load_environment_validates_env_config_subclass_sections(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -3099,6 +3161,36 @@ def test_load_environment_composes_component_package_without_root_loader(
     assert env.harness.config.max_turns == 3
 
 
+def test_load_environment_component_package_accepts_top_level_config_shorthand(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "component_only_taskset_shorthand"
+    module = types.ModuleType(module_name)
+
+    class LocalTasksetConfig(TasksetConfig):
+        answer: str = "configured"
+
+    class LocalTaskset(Taskset[LocalTasksetConfig]):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            return [{"prompt": [], "answer": f"{split}:{self.config.answer}"}]
+
+    def load_taskset(config: LocalTasksetConfig) -> LocalTaskset:
+        return LocalTaskset(config=config)
+
+    module.load_taskset = load_taskset
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    env = vf.load_environment(
+        "component-only-taskset-shorthand",
+        taskset={"answer": "top-level"},
+        harness={"max_turns": 2},
+    )
+
+    assert env.taskset.get_dataset()[0]["answer"] == "train:top-level"
+    assert type(env.harness) is Harness
+    assert env.harness.config.max_turns == 2
+
+
 def test_load_environment_delegates_missing_child_loaders_by_config_id(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

verifiers-0.1.15.dev159/tests/test_v1_replay_harness.py

@@ -0,0 +1,327 @@
+import json
+from pathlib import Path
+
+import pytest
+
+import verifiers as vf
+from harnesses import ReplayHarness
+from tasksets import ReplayTaskset, ReplayTasksetConfig
+from tasksets.replay import replay_task_record
+
+
+class NoModelClient:
+    def __init__(self) -> None:
+        self.requests = 0
+
+    async def get_response(self, **kwargs: object) -> object:
+        _ = kwargs
+        self.requests += 1
+        raise AssertionError("ReplayHarness must not request model completions.")
+
+
+class InlineReplayTaskset(ReplayTaskset):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+        if split == "eval":
+            return []
+        return [
+            {
+                "messages": [
+                    {"role": "user", "content": "Reverse abc."},
+                    {"role": "assistant", "content": "cba"},
+                    {"role": "user", "content": "Now uppercase it."},
+                    {
+                        "role": "assistant",
+                        "content": "CBA",
+                        "reasoning_content": "uppercased the prior answer",
+                    },
+                    {"role": "user", "content": "Thanks."},
+                ],
+            }
+        ]
+
+
+class ManyTurnReplayTaskset(ReplayTaskset):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+        if split == "eval":
+            return []
+        messages = []
+        for index in range(11):
+            messages.append({"role": "user", "content": f"Turn {index}?"})
+            messages.append({"role": "assistant", "content": f"reply {index}"})
+        return [{"messages": messages}]
+
+
+@pytest.mark.asyncio
+async def test_replay_harness_prints_assistant_messages_into_trajectory() -> None:
+    env = vf.Env(
+        taskset=InlineReplayTaskset(),
+        harness=ReplayHarness(config=vf.HarnessConfig()),
+    )
+    client = NoModelClient()
+
+    state = await env.rollout(
+        dict(env.get_dataset()[0]),
+        client=client,
+        model="mock-model",
+    )
+
+    assert client.requests == 0
+    assert state["stop_condition"] == "replayed_messages"
+    assert state["num_model_requests"] == 2
+    assert state["prompt"] == [{"role": "user", "content": "Reverse abc."}]
+    assert state["completion"] == [
+        {"role": "assistant", "content": "cba"},
+        {"role": "user", "content": "Now uppercase it."},
+        {
+            "role": "assistant",
+            "content": "CBA",
+            "reasoning_content": "uppercased the prior answer",
+        },
+    ]
+    assert state["completion"][-1]["role"] == "assistant"
+
+    first, second = state["trajectory"]
+    assert first["prompt"] == [{"role": "user", "content": "Reverse abc."}]
+    assert first["completion"] == [{"role": "assistant", "content": "cba"}]
+    assert first["tokens"] is None
+    assert "tokens" not in first["response"]["message"]
+
+    assert second["prompt"] == [
+        {"role": "user", "content": "Reverse abc."},
+        {"role": "assistant", "content": "cba"},
+        {"role": "user", "content": "Now uppercase it."},
+    ]
+    assert second["completion"] == [
+        {
+            "role": "assistant",
+            "content": "CBA",
+            "reasoning_content": "uppercased the prior answer",
+        }
+    ]
+    assert second["tokens"] is None
+    assert "tokens" not in second["response"]["message"]
+
+
+@pytest.mark.asyncio
+async def test_replay_harness_marks_partial_replay_as_truncated() -> None:
+    env = vf.Env(
+        taskset=InlineReplayTaskset(),
+        harness=ReplayHarness(config=vf.HarnessConfig(max_turns=1)),
+    )
+
+    state = await env.rollout(
+        dict(env.get_dataset()[0]),
+        client=NoModelClient(),
+        model="mock-model",
+    )
+
+    assert state["stop_condition"] == "max_turns_reached"
+    assert state["is_truncated"] is True
+    assert state["num_model_requests"] == 1
+    assert state["completion"] == [{"role": "assistant", "content": "cba"}]
+    step = state["trajectory"][0]
+    assert step["is_truncated"] is True
+    assert step["response"]["message"]["is_truncated"] is True
+
+
+@pytest.mark.asyncio
+async def test_replay_harness_defaults_to_all_assistant_messages() -> None:
+    assert vf.HarnessConfig().max_turns == -1
+    env = vf.Env(
+        taskset=ManyTurnReplayTaskset(),
+        harness=ReplayHarness(config=vf.HarnessConfig()),
+    )
+
+    state = await env.rollout(
+        dict(env.get_dataset()[0]),
+        client=NoModelClient(),
+        model="mock-model",
+    )
+
+    assert state["stop_condition"] == "replayed_messages"
+    assert state["is_truncated"] is False
+    assert state["num_model_requests"] == 11
+    assert len(state["trajectory"]) == 11
+    assert state["completion"][-1] == {"role": "assistant", "content": "reply 10"}
+
+
+def test_replay_taskset_loads_configured_local_jsonl_data(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "examples.jsonl").write_text(
+        "\n".join(
+            [
+                json.dumps(
+                    {
+                        "messages": [
+                            {"role": "user", "content": "Say ok."},
+                            {"role": "assistant", "content": "ok"},
+                        ]
+                    }
+                ),
+                json.dumps(
+                    {
+                        "messages": [
+                            {"role": "user", "content": "Say yes."},
+                            {"role": "assistant", "content": "yes"},
+                        ]
+                    }
+                ),
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    assert taskset.load_tasks() == [
+        {
+            "messages": [
+                {"role": "user", "content": "Say ok."},
+                {"role": "assistant", "content": "ok"},
+            ]
+        },
+        {
+            "messages": [
+                {"role": "user", "content": "Say yes."},
+                {"role": "assistant", "content": "yes"},
+            ]
+        },
+    ]
+
+
+def test_replay_taskset_loads_subclass_local_jsonl_data(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "example.jsonl").write_text(
+        json.dumps(
+            {
+                "messages": [
+                    {"role": "user", "content": "Say ok."},
+                    {"role": "assistant", "content": "ok"},
+                ]
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    local_taskset_type = type(
+        "LocalReplayTaskset",
+        (ReplayTaskset,),
+        {"data_dir": str(data_dir)},
+    )
+    taskset = local_taskset_type(config=ReplayTasksetConfig())
+
+    assert taskset.load_tasks() == [
+        {
+            "messages": [
+                {"role": "user", "content": "Say ok."},
+                {"role": "assistant", "content": "ok"},
+            ]
+        }
+    ]
+
+
+def test_replay_taskset_rejects_missing_local_source() -> None:
+    taskset = ReplayTaskset(config=ReplayTasksetConfig())
+
+    with pytest.raises(FileNotFoundError, match="requires dataset or data_dir"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_conflicting_sources(tmp_path: Path) -> None:
+    taskset = ReplayTaskset(
+        config=ReplayTasksetConfig(dataset="owner/dataset", data_dir=str(tmp_path))
+    )
+
+    with pytest.raises(ValueError, match="cannot set both dataset and data_dir"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_empty_local_data_dir(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    with pytest.raises(FileNotFoundError, match="must contain at least one JSONL"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_json_files(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "example.json").write_text(
+        json.dumps(
+            {
+                "messages": [
+                    {"role": "user", "content": "Say ok."},
+                    {"role": "assistant", "content": "ok"},
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    with pytest.raises(ValueError, match=r"accepts only \.jsonl files"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_non_object_jsonl_rows(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "example.jsonl").write_text("[]\n", encoding="utf-8")
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    with pytest.raises(TypeError, match="example.jsonl:1 must contain one JSON object"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_canonicalizes_messages() -> None:
+    task = replay_task_record(
+        {
+            "messages": [
+                {"role": "user", "content": "Use the tool."},
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": "call_1",
+                            "type": "function",
+                            "function": {
+                                "name": "search",
+                                "arguments": {"query": "abc"},
+                            },
+                        }
+                    ],
+                },
+            ]
+        }
+    )
+
+    assert task["messages"] == [
+        {"role": "user", "content": "Use the tool."},
+        {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "name": "search",
+                    "arguments": '{"query": "abc"}',
+                }
+            ],
+        },
+    ]
+
+
+def test_replay_taskset_rejects_invalid_messages() -> None:
+    with pytest.raises(TypeError, match="messages must be a list"):
+        replay_task_record({"messages": "not a transcript"})
+
+    with pytest.raises(ValueError, match="Unknown role"):
+        replay_task_record({"messages": [{"role": "assistantish", "content": "no"}]})

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev159}/tests/test_v1_runtime_lifecycle.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 
 import verifiers as vf
 from verifiers.clients import Client
-from verifiers.types import ClientConfig
+from verifiers.types import ClientConfig, Messages
 from verifiers.types import Response, ResponseMessage, ToolCall
 from verifiers.types import Tool
 from verifiers.types import Usage
@@ -98,6 +98,18 @@ class CapturingModelClient(FakeModelClient):
         return await super().get_response(**kwargs)
 
 
+class BlockingModelClient(CapturingModelClient):
+    def __init__(self, responses: list[Response]):
+        super().__init__(responses)
+        self.entered = asyncio.Event()
+        self.release = asyncio.Event()
+
+    async def get_response(self, **kwargs: object) -> Response:
+        self.entered.set()
+        await self.release.wait()
+        return await super().get_response(**kwargs)
+
+
 class FakeCreateSandboxRequest:
     def __init__(self, **kwargs: object):
         self.kwargs = kwargs
@@ -537,6 +549,37 @@ async def endpoint_trajectory_program(task, state):
     return state
 
 
+async def concurrent_endpoint_program(task, state):
+    _ = task
+    root = state["endpoint_root_url"].rstrip("/")
+    config = state.get_endpoint_config(api="chat")
+    endpoint_client = cast(OpenAI, state.get_client(api="chat", sync=True))
+    api_key = endpoint_client.api_key
+    endpoint_client.close()
+
+    def post_chat(content: str) -> dict[str, object]:
+        payload = {
+            "model": config.model,
+            "messages": [{"role": "user", "content": content}],
+        }
+        request = urllib.request.Request(
+            f"{root}/v1/chat/completions",
+            data=json.dumps(payload).encode(),
+            headers={
+                "content-type": "application/json",
+                "Authorization": f"Bearer {api_key}",
+            },
+        )
+        with urllib.request.urlopen(request) as response:
+            return json.loads(response.read().decode())
+
+    state["endpoint_concurrent_responses"] = await asyncio.gather(
+        asyncio.to_thread(post_chat, "first"),
+        asyncio.to_thread(post_chat, "second"),
+    )
+    return state
+
+
 async def mcp_proxy_program(task, state):
     _ = task
     from mcp import ClientSession, StdioServerParameters
@@ -683,6 +726,7 @@ for _name, _value in {
     "child_reads_program_sandbox": child_reads_program_sandbox,
     "endpoint_program": endpoint_program,
     "endpoint_trajectory_program": endpoint_trajectory_program,
+    "concurrent_endpoint_program": concurrent_endpoint_program,
     "mcp_proxy_program": mcp_proxy_program,
     "child_program": child_program,
     "parent_program": parent_program,
@@ -808,6 +852,36 @@ async def test_endpoint_request_can_hide_internal_model_call_from_trajectory() -
     )
 
 
+@pytest.mark.asyncio
+async def test_endpoint_max_turns_counts_inflight_visible_requests() -> None:
+    client = BlockingModelClient(
+        [fake_response("allowed"), fake_response("unexpected")]
+    )
+    harness = make_harness(
+        program={"fn": program_ref("concurrent_endpoint_program")},
+        client=client,
+        model="test-model",
+        max_turns=1,
+    )
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    run_task = asyncio.create_task(harness.run(task))
+    await asyncio.wait_for(client.entered.wait(), timeout=1.0)
+    await asyncio.sleep(0.05)
+    client.release.set()
+    state = await run_task
+    await harness.teardown()
+
+    contents = [
+        response["choices"][0]["message"]["content"]
+        for response in state["endpoint_concurrent_responses"]
+    ]
+    assert sorted(contents) == ["", "allowed"]
+    assert len(client.requests) == 1
+    assert len(state["trajectory"]) == 1
+    assert state["stop_condition"] == "max_turns_reached"
+
+
 @pytest.mark.asyncio
 async def test_state_helpers_load_runtime_tools_while_rollout_is_active() -> None:
     harness = make_harness(
@@ -950,6 +1024,38 @@ async def test_base_program_submits_system_prompt_before_prompt() -> None:
     assert state["completion"][-1]["content"] == "ok"
 
 
+@pytest.mark.asyncio
+async def test_base_program_max_turns_uses_stop_condition() -> None:
+    client = CapturingModelClient(
+        [fake_response(content="done"), fake_response(content="unexpected")]
+    )
+    harness = make_harness(client=cast(Client, client), model="fake", max_turns=1)
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    state = await harness.run(task)
+
+    assert len(client.requests) == 1
+    assert len(state["trajectory"]) == 1
+    assert state["stop_condition"] == "max_turns_reached"
+
+
+@pytest.mark.asyncio
+async def test_model_request_reservation_released_when_client_resolution_fails() -> (
+    None
+):
+    harness = make_harness(model="fake", max_turns=1)
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+    state = await harness.setup_state(task, vf.State.for_task(task))
+
+    with pytest.raises(RuntimeError, match="no model client"):
+        await harness.runtime.submit_model_request(
+            cast(Messages, state["prompt"]), task, state
+        )
+
+    assert harness.runtime.visible_model_requests(state) == 0
+    assert state["trajectory"] == []
+
+
 @pytest.mark.asyncio
 async def test_taskset_setup_initializes_base_harness_prompt_and_sampling() -> None:
     class SetupRuntimeTaskset(vf.Taskset):