verifiers 0.1.15.dev166__tar.gz → 0.1.15.dev168__tar.gz

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev166
+Version: 0.1.15.dev168
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_composable_env.py

@@ -187,6 +187,28 @@ def test_taskset_take():
     assert isinstance(taken, MockSandboxTaskSet)
 
 
+def test_composable_env_eval_inputs_can_shuffle_taskset_dataset():
+    taskset = MockTaskSet(dataset=_make_dataset(6), name="math")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(run_command="true"),
+    )
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=11,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=11).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]
+
+
 def test_taskset_repr():
     ts = MockTaskSet(dataset=_make_dataset(), name="mytest")
     assert "mytest" in repr(ts)

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_environment.py

@@ -697,6 +697,30 @@ class TestMaybeRetry:
         error_data = rollout_outputs[0]["error"]
         assert "InfraError" == error_data["error"]
 
+    @pytest.mark.asyncio
+    async def test_retries_serialized_infra_error_subclass(self):
+        """A serialized InfraError subclass (e.g. SandboxError) in returned state
+        must trigger retry.
+
+        The v1 harness serializes state["error"] to ErrorData before maybe_retry
+        inspects it, so matching must be subclass-aware (rebuild concrete error +
+        isinstance) — base-name substring matching missed SandboxError, which is
+        an InfraError and should be retried.
+        """
+        from verifiers.utils.async_utils import maybe_retry
+        from verifiers.utils.error_utils import error_data
+
+        serialized = error_data(vf.SandboxError("Program file upload failed"))
+        calls = {"n": 0}
+
+        async def attempt():
+            calls["n"] += 1
+            return {"error": serialized}
+
+        result = await maybe_retry(attempt, max_retries=2, initial=0.0, max_wait=0.0)()
+        assert calls["n"] == 3  # 1 initial + 2 retries (InfraError is retryable)
+        assert result["error"] == serialized  # last result returned after exhaustion
+
 
 class TestEmptyModelResponseErrors:
     """Test cases for empty and invalid model response error handling."""

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_environment_extra.py

@@ -302,6 +302,30 @@ def test_evaluate_fallback_and_repeat(mock_client, make_dummy_env, make_input):
     assert len(states) == 2 * 2
 
 
+def test_get_eval_inputs_shuffles_before_take_and_repeat(mock_client, make_dummy_env):
+    ds = Dataset.from_dict(
+        {
+            "question": [f"q{i}" for i in range(6)],
+            "answer": [f"a{i}" for i in range(6)],
+        }
+    )
+    env = make_dummy_env(mock_client, dataset=ds)
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=123,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=123).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]
+
+
 @pytest.mark.asyncio
 async def test_generate_inside_running_loop(mock_client, make_dummy_env, make_input):
     env = make_dummy_env(mock_client)

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_eval_cli.py

@@ -72,6 +72,8 @@ def run_cli(make_metadata, make_state, make_input):
             "headers_from_state": None,
             "num_examples": 1,
             "rollouts_per_example": 1,
+            "shuffle": False,
+            "shuffle_seed": None,
             "max_concurrent": 1,
             "independent_scoring": False,
             "max_tokens": 42,
@@ -177,6 +179,20 @@ def test_parse_args_rejects_unknown_eval_flags():
         vf_eval.parse_args(["env1", "--unknown-flag", "value"])
 
 
+def test_cli_shuffle_defaults_seed_when_enabled(monkeypatch, run_cli):
+    captured = run_cli(
+        monkeypatch,
+        {
+            "shuffle": True,
+            "shuffle_seed": None,
+        },
+    )
+
+    config = captured["configs"][0]
+    assert config.shuffle is True
+    assert config.shuffle_seed == 0
+
+
 def test_cli_v1_env_config_overrides_preserve_env_args_config(
     tmp_path: Path, monkeypatch, run_cli
 ):
@@ -1188,6 +1204,15 @@ def test_cli_toml_per_env_rollouts_per_example(monkeypatch, run_cli):
     assert configs[1].rollouts_per_example == 5
 
 
+def test_cli_toml_per_env_shuffle(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write('[[eval]]\nenv_id = "env1"\nshuffle = true\nshuffle_seed = 321\n')
+        f.flush()
+        captured = run_cli(monkeypatch, {"env_id_or_config": f.name})
+
+    assert captured["configs"][0].shuffle_seed == 321
+
+
 def test_cli_toml_per_eval_settings_used(monkeypatch, run_cli):
     """TOML per-eval settings are used (CLI args ignored when using config)."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_path_utils.py

@@ -70,6 +70,53 @@ def test_find_latest_incomplete_eval_results_path_returns_none_when_no_match(
     assert result is None
 
 
+def test_find_latest_incomplete_eval_results_path_matches_shuffle_metadata(
+    tmp_path: Path, monkeypatch
+):
+    env_id = "dummy-env"
+    model = "openai/gpt-4.1-mini"
+    runs_dir = tmp_path / "outputs" / "evals" / f"{env_id}--{model.replace('/', '--')}"
+
+    matching_run = runs_dir / "matching"
+    wrong_seed_run = runs_dir / "wrong-seed"
+    for run in [matching_run, wrong_seed_run]:
+        run.mkdir(parents=True)
+        (run / "results.jsonl").write_text('{"example_id":0}\n', encoding="utf-8")
+
+    matching_run.joinpath("metadata.json").write_text(
+        (
+            '{"env_id":"dummy-env","model":"openai/gpt-4.1-mini",'
+            '"num_examples":4,"rollouts_per_example":1,'
+            '"shuffle":true,"shuffle_seed":123}'
+        ),
+        encoding="utf-8",
+    )
+    wrong_seed_run.joinpath("metadata.json").write_text(
+        (
+            '{"env_id":"dummy-env","model":"openai/gpt-4.1-mini",'
+            '"num_examples":4,"rollouts_per_example":1,'
+            '"shuffle":true,"shuffle_seed":456}'
+        ),
+        encoding="utf-8",
+    )
+
+    os.utime(matching_run, (1, 1))
+    os.utime(wrong_seed_run, (2, 2))
+    monkeypatch.chdir(tmp_path)
+
+    result = find_latest_incomplete_eval_results_path(
+        env_id=env_id,
+        model=model,
+        num_examples=4,
+        rollouts_per_example=1,
+        shuffle=True,
+        shuffle_seed=123,
+        env_dir_path=str(tmp_path / "environments"),
+    )
+
+    assert result.resolve() == matching_run
+
+
 def test_get_eval_runs_dir_uses_name_as_result_label(tmp_path: Path):
     runs_dir = get_eval_runs_dir(
         env_id="dummy-env",

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_save_utils.py

@@ -31,6 +31,7 @@ from verifiers.utils.save_utils import (
     load_outputs,
     make_serializable,
     save_new_outputs,
+    save_metadata,
     states_to_outputs,
     truncate_malformed_trailing_line,
     validate_resume_metadata,
@@ -199,6 +200,37 @@ class TestSavingMetadata:
             metadata["base_url"] == "http://localhost:8000/v1,http://localhost:8001/v1"
         )
 
+    def test_save_metadata_records_default_shuffle_seed_for_resume(
+        self, tmp_path: Path
+    ):
+        results_path = tmp_path / "results"
+        builder = GenerateOutputsBuilder(
+            env_id="math-env",
+            env_args={},
+            model="test-model",
+            client=ClientConfig(api_base_url="http://localhost:8000/v1"),
+            num_examples=3,
+            rollouts_per_example=2,
+            state_columns=[],
+            sampling_args={},
+            results_path=results_path,
+            shuffle=True,
+        )
+        save_metadata(builder.build_metadata(), results_path)
+
+        assert (
+            json.loads((results_path / "metadata.json").read_text())["shuffle_seed"]
+            == 0
+        )
+        validate_resume_metadata(
+            results_path=results_path,
+            env_id="math-env",
+            model="test-model",
+            num_examples=3,
+            rollouts_per_example=2,
+            shuffle=True,
+        )
+
 
 class TestSavingResults:
     def test_response_usage_tokens_prompt_completion(self):
@@ -595,6 +627,35 @@ class TestResumeMetadataValidation:
                 rollouts_per_example=2,
             )
 
+    def test_validate_resume_metadata_rejects_shuffle_mismatch(self, tmp_path: Path):
+        results_path = tmp_path / "results"
+        results_path.mkdir()
+        metadata_path = results_path / "metadata.json"
+        metadata_path.write_text(
+            json.dumps(
+                {
+                    "env_id": "math-env",
+                    "model": "test-model",
+                    "num_examples": 3,
+                    "rollouts_per_example": 2,
+                    "shuffle": True,
+                    "shuffle_seed": 123,
+                }
+            ),
+            encoding="utf-8",
+        )
+
+        with pytest.raises(ValueError, match="shuffle_seed"):
+            validate_resume_metadata(
+                results_path=results_path,
+                env_id="math-env",
+                model="test-model",
+                num_examples=3,
+                rollouts_per_example=2,
+                shuffle=True,
+                shuffle_seed=456,
+            )
+
 
 class TestBuilderPassAtK:
     """Tests for pass@k integration in GenerateOutputsBuilder."""

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_trajectory_processing.py

@@ -282,6 +282,49 @@ async def test_parsed_prompt_attribution_survives_v1_assert_serializable():
     State({"trajectory": [step]}).assert_serializable()
 
 
+def test_assert_serializable_accepts_msgpack_sidecars_rejects_unknown():
+    """The ``assert_serializable`` json.dumps gate must accept exactly what the
+    trainer transport (msgpack) accepts, while staying strict otherwise.
+
+    Trajectory token steps carry sidecars that are non-JSON by design and reach
+    the trainer via msgpack, not JSON: the renderer ``MultiModalData`` (a
+    dataclass holding numpy pixel arrays) and ``routed_experts`` (a raw
+    ``memoryview`` buffer). Both must clear the gate; any other
+    non-serializable object must still raise.
+    """
+    import dataclasses
+
+    import numpy as np
+
+    @dataclasses.dataclass
+    class _FakeMultiModalData:
+        mm_hashes: dict
+        mm_items: dict
+        mm_placeholders: dict
+
+    mm = _FakeMultiModalData(
+        mm_hashes={"image": ["h1"]},
+        mm_items={"image": [np.zeros((2, 2), dtype=np.uint8)]},
+        mm_placeholders={"image": [{"offset": 0, "length": 4}]},
+    )
+    step = {
+        "tokens": {
+            "prompt_ids": [1, 2],
+            "multi_modal_data": mm,
+            "routed_experts": {"data": memoryview(b"abc"), "shape": [3], "start": 0},
+        }
+    }
+    # Must not raise: both sidecars are msgpack-transported, not JSON.
+    State({"trajectory": [step]}).assert_serializable()
+
+    # A genuinely non-serializable object must still be rejected.
+    class _Unknown:
+        pass
+
+    with pytest.raises(TypeError):
+        State({"trajectory": [{"tokens": _Unknown()}]}).assert_serializable()
+
+
 def test_process_trajectory_steps_for_training(make_input):
     """Test processing trajectory steps into training examples."""
     state1 = State(

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_v1_runtime_lifecycle.py

@@ -110,6 +110,15 @@ class BlockingModelClient(CapturingModelClient):
         return await super().get_response(**kwargs)
 
 
+class RaisingModelClient:
+    def __init__(self, error: vf.Error):
+        self.error = error
+
+    async def get_response(self, **kwargs: object) -> Response:
+        _ = kwargs
+        raise self.error
+
+
 class FakeCreateSandboxRequest:
     def __init__(self, **kwargs: object):
         self.kwargs = kwargs
@@ -517,6 +526,31 @@ async def endpoint_program(task, state):
     }
 
 
+async def endpoint_model_error_program(task, state):
+    _ = task
+    root = state["endpoint_root_url"].rstrip("/")
+    endpoint_client = cast(OpenAI, state.get_client(api="chat", sync=True))
+    auth_headers = {"Authorization": f"Bearer {endpoint_client.api_key}"}
+    endpoint_client.close()
+
+    def post_model() -> None:
+        request = urllib.request.Request(
+            f"{root}/vf/model",
+            data=json.dumps(
+                {"messages": [{"role": "user", "content": "too long"}]}
+            ).encode(),
+            headers={"content-type": "application/json", **auth_headers},
+        )
+        with urllib.request.urlopen(request):
+            pass
+
+    try:
+        await asyncio.to_thread(post_model)
+    except Exception as exc:
+        raise vf.SandboxError("Sandbox command failed") from exc
+    raise AssertionError("Expected /vf/model to fail")
+
+
 async def endpoint_trajectory_program(task, state):
     _ = task
     root = state["endpoint_root_url"].rstrip("/")
@@ -725,6 +759,7 @@ for _name, _value in {
     "initialize_from_taskset": initialize_from_taskset,
     "child_reads_program_sandbox": child_reads_program_sandbox,
     "endpoint_program": endpoint_program,
+    "endpoint_model_error_program": endpoint_model_error_program,
     "endpoint_trajectory_program": endpoint_trajectory_program,
     "concurrent_endpoint_program": concurrent_endpoint_program,
     "mcp_proxy_program": mcp_proxy_program,
@@ -827,6 +862,41 @@ async def test_endpoint_exposes_tool_user_and_stop_surfaces() -> None:
     assert "endpoint_root_url" not in state
 
 
+@pytest.mark.asyncio
+async def test_vf_model_bridge_preserves_overlong_prompt_error() -> None:
+    harness = make_harness(
+        program={"fn": program_ref("endpoint_model_error_program")},
+        model="test-model",
+        client=RaisingModelClient(vf.OverlongPromptError("too long")),
+    )
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    state = await harness.run(task)
+    await harness.teardown()
+
+    assert state["prompt_too_long"] is True
+    assert state["is_truncated"] is True
+    assert state["stop_condition"] == "prompt_too_long"
+    assert state.get("error") is None
+
+
+@pytest.mark.asyncio
+async def test_vf_model_bridge_preserves_model_error() -> None:
+    harness = make_harness(
+        program={"fn": program_ref("endpoint_model_error_program")},
+        model="test-model",
+        client=RaisingModelClient(vf.ModelError("model failed")),
+    )
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    state = await harness.run(task)
+    await harness.teardown()
+
+    assert state["stop_condition"] == "has_error"
+    assert state["error"]["error"] == "ModelError"
+    assert "SandboxError" not in state["error"]["error_chain_str"]
+
+
 @pytest.mark.asyncio
 async def test_endpoint_request_can_hide_internal_model_call_from_trajectory() -> None:
     client = FakeModelClient([fake_response("hidden"), fake_response("shown")])
@@ -1462,6 +1532,70 @@ async def test_create_sandbox_cleans_up_wait_failure_with_retry(
     assert client.delete_calls == 2
 
 
+@pytest.mark.asyncio
+async def test_upload_program_files_retries_transient_transfer_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_fake_sandboxes(monkeypatch)
+    disable_sandbox_retry_sleep(monkeypatch)
+
+    class FlakyUploadClient:
+        calls = 0
+
+        async def upload_bytes(self, *args: object, **kwargs: object) -> None:
+            _ = args, kwargs
+            self.calls += 1
+            if self.calls == 1:
+                raise FakeAPIError("Upload failed: ")
+
+    client = FlakyUploadClient()
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+    state = vf.State.for_task(task)
+
+    await sandbox_utils.upload_program_files(
+        cast(sandbox_utils.SandboxClient, client),
+        "sbx-upload",
+        {"files": {"/tmp/file.txt": "content"}},
+        task,
+        state,
+        Runtime(),
+    )
+
+    assert client.calls == 2
+
+
+@pytest.mark.asyncio
+async def test_upload_program_files_does_not_retry_non_transient_api_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_fake_sandboxes(monkeypatch)
+    disable_sandbox_retry_sleep(monkeypatch)
+
+    class FailingUploadClient:
+        calls = 0
+
+        async def upload_bytes(self, *args: object, **kwargs: object) -> None:
+            _ = args, kwargs
+            self.calls += 1
+            raise FakeAPIError("Upload failed: HTTP 400: bad request")
+
+    client = FailingUploadClient()
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+    state = vf.State.for_task(task)
+
+    with pytest.raises(vf.SandboxError, match="HTTP 400"):
+        await sandbox_utils.upload_program_files(
+            cast(sandbox_utils.SandboxClient, client),
+            "sbx-upload",
+            {"files": {"/tmp/file.txt": "content"}},
+            task,
+            state,
+            Runtime(),
+        )
+
+    assert client.calls == 1
+
+
 @pytest.mark.asyncio
 async def test_create_sandbox_cancellation_deletes_late_provider_result(
     monkeypatch: pytest.MonkeyPatch,
@@ -1603,8 +1737,8 @@ async def test_sandbox_base_program_max_turns_zero_is_unbounded(
     config_path.write_text(json.dumps({"max_turns": 0}))
     namespace["RUNNER_CONFIG_PATH"] = str(config_path)
 
-    async def create_model_message(state, messages, client):
-        _ = state, messages, client
+    async def create_model_message(state, messages):
+        _ = state, messages
         return {"role": "assistant", "content": "done"}
 
     async def call_user(state, messages):
@@ -1621,12 +1755,55 @@ async def test_sandbox_base_program_max_turns_zero_is_unbounded(
 
     state = {"prompt": [{"role": "user", "content": "hi"}], "runtime": {}}
     run_base = cast(Any, namespace["run_base"])
-    result = await run_base({}, state, object())
+    result = await run_base({}, state)
 
     assert result["completion"] == [{"role": "assistant", "content": "done"}]
     assert result["stop_condition"] == "no_tools"
 
 
+@pytest.mark.asyncio
+async def test_sandbox_base_program_model_call_uses_vf_model_bridge() -> None:
+    namespace: dict[str, object] = {}
+    source = runner_source().rsplit("asyncio.run(main())", 1)[0]
+    exec(source, namespace)
+
+    posted: list[tuple[str, Any, object]] = []
+
+    async def vf_post(state, path, payload, timeout=None):
+        _ = state
+        posted.append((path, payload, timeout))
+        return {"message": {"role": "assistant", "content": "ok"}}
+
+    namespace["vf_post"] = vf_post
+    create_model_message = cast(Any, namespace["create_model_message"])
+
+    # Canonical Messages (incl. an image content part) are sent unchanged over the
+    # /vf/model bridge; the host owns client resolution + tokenization and returns
+    # the assistant message.
+    messages = [
+        {"role": "user", "content": "hi"},
+        {
+            "role": "tool",
+            "tool_call_id": "call_1",
+            "content": [
+                {"type": "text", "text": "shot"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/png;base64,AAA"},
+                },
+            ],
+        },
+    ]
+    message = await create_model_message({"runtime": {}}, messages)
+
+    assert message == {"role": "assistant", "content": "ok"}
+    assert len(posted) == 1
+    path, payload, timeout = posted[0]
+    assert path == "model"
+    assert payload["messages"] == messages  # image part preserved verbatim
+    assert timeout is None
+
+
 def test_sandbox_program_patch_cannot_set_lifecycle_fields() -> None:
     state = vf.State.for_task(vf.Task({"prompt": []}).freeze())
 

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/tests/test_v1_taskset_utils.py

@@ -4,7 +4,7 @@ import types
 
 from datasets import Dataset
 
-from verifiers.v1 import Taskset
+from verifiers.v1 import Env, Taskset
 from verifiers.v1.utils.taskset_utils import dataset_from_result, discover_sibling_dir
 
 
@@ -118,3 +118,25 @@ def test_taskset_skips_empty_skills_dir(tmp_path, monkeypatch) -> None:
 
     assert taskset.get_skills_dir() == skills_dir
     assert taskset.get_upload_dirs() == {"skills": skills_dir}
+
+
+def test_v1_env_eval_inputs_can_shuffle_taskset_dataset() -> None:
+    class DemoTaskset(Taskset):
+        def load_tasks(self, split: str = "train"):
+            return [{"question": f"Reverse {i}.", "answer": str(i)} for i in range(6)]
+
+    env = Env(taskset=DemoTaskset())
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=7,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=7).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/verifiers/clients/anthropic_messages_client.py

@@ -214,15 +214,15 @@ class AnthropicMessagesClient(
             return {}
 
         def build_tool_result_block(message: ToolMessage) -> ToolResultBlockParam:
+            if isinstance(message.content, str):
+                result_content: Any = message.content
+            else:
+                # Keep images: image_url parts -> Anthropic image blocks (not "[image]" text).
+                result_content = normalize_anthropic_content(message.content)
             return ToolResultBlockParam(
                 type="tool_result",
                 tool_use_id=message.tool_call_id,
-                content=cast(
-                    Any,
-                    message.content
-                    if isinstance(message.content, str)
-                    else " ".join(content_to_text_chunks(message.content)),
-                ),
+                content=cast(Any, result_content),
             )
 
         def from_chat_message(message: Message) -> AnthropicMessageParam | None:

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev168}/verifiers/clients/openai_responses_client.py

@@ -156,8 +156,8 @@ class OpenAIResponsesClient(
             if isinstance(message, ToolMessage):
                 output = message.content
                 if not isinstance(output, str):
-                    text = content_to_text(output)
-                    output = text if text else str(output)
+                    # Keep images: image_url parts -> Responses input_image (not text).
+                    output = normalize_message_content(output)
                 return [
                     {
                         "type": "function_call_output",