verifiers 0.1.12.dev1__tar.gz → 0.1.12.dev3__tar.gz

{verifiers-0.1.12.dev1 → verifiers-0.1.12.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.12.dev1
+Version: 0.1.12.dev3
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -34,12 +34,14 @@ Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: numpy
 Requires-Dist: openai-agents>=0.0.7
 Requires-Dist: openai>=1.108.1
-Requires-Dist: prime-sandboxes>=0.2.16
-Requires-Dist: prime-tunnel>=0.1.4
+Requires-Dist: prime-sandboxes>=0.2.19
+Requires-Dist: prime-tunnel>=0.1.5
 Requires-Dist: pydantic>=2.11.9
 Requires-Dist: pyzmq>=27.1.0
+Requires-Dist: regex<2026.4.4
 Requires-Dist: requests
 Requires-Dist: rich
+Requires-Dist: setproctitle>=1.3.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: textual
 Requires-Dist: tomli; python_version < '3.11'

{verifiers-0.1.12.dev1 → verifiers-0.1.12.dev3}/pyproject.toml

@@ -37,8 +37,8 @@ dependencies = [
     "nest-asyncio>=1.6.0", # for jupyter notebooks
     "openai>=1.108.1",
     "openai-agents>=0.0.7",
-    "prime-tunnel>=0.1.4",
-    "prime-sandboxes>=0.2.16",
+    "prime-tunnel>=0.1.5",
+    "prime-sandboxes>=0.2.19",
     "pydantic>=2.11.9",
     "requests",
     "rich",
@@ -51,6 +51,8 @@ dependencies = [
     "pyzmq>=27.1.0",
     "msgpack>=1.1.2",
     "aiolimiter>=1.2.1",
+    "setproctitle>=1.3.0",
+    "regex<2026.4.4",  # 2026.4.4 missing cp312/cp313 wheels
 ]
 
 [dependency-groups]
@@ -104,7 +106,7 @@ rl = [
 
 [tool.uv]
 preview = true
-required-version = "<0.11.0"
+required-version = ">=0.11.1"
 
 [tool.uv.extra-build-dependencies]
 flash-attn = [{ requirement = "torch", match-runtime = true }]

verifiers-0.1.12.dev3/tests/test_composable_env.py

@@ -0,0 +1,260 @@
+"""Tests for the composable architecture: Task, TaskSet, SandboxTaskSet, SandboxSpec."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+import verifiers as vf
+from verifiers.envs.experimental.composable import (
+    ComposableEnv,
+    Harness,
+    SandboxSpec,
+    SandboxTaskSet,
+    Task,
+    TaskSet,
+)
+
+
+# ── Mock Rubrics ──────────────────────────────────────────────────────
+
+
+class MockSandboxRubric(vf.Rubric):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_reward_func(self.solved)
+
+    async def solved(self, state, **kwargs) -> float:
+        return 1.0 if state.get("test_output") == "PASS" else 0.0
+
+
+class MockMathRubric(vf.Rubric):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_reward_func(self.correct)
+
+    async def correct(self, state, **kwargs) -> float:
+        return 1.0 if state.get("info", {}).get("id") == 0 else 0.0
+
+
+# ── Mock TaskSets ───────────────────────────────────────────────────────
+
+
+class MockSandboxTaskSet(SandboxTaskSet):
+    """SandboxTaskSet for testing."""
+
+    def get_instruction(self, info):
+        return f"Fix bug #{info.get('id', 0)}"
+
+    def get_sandbox_spec(self, info):
+        return SandboxSpec(image="python:3.11-slim", cpu_cores=2, memory_gb=2)
+
+    def get_rubric(self):
+        return MockSandboxRubric()
+
+    def get_workdir(self, info):
+        return "/testbed"
+
+    def get_env_vars(self):
+        return {"FOO": "bar"}
+
+
+class MockTaskSet(TaskSet):
+    """Plain TaskSet (no sandbox) for testing."""
+
+    def get_instruction(self, info):
+        return info.get("question", "")
+
+    def get_rubric(self):
+        return MockMathRubric()
+
+
+def _make_dataset(n=3):
+    from datasets import Dataset
+
+    return Dataset.from_dict(
+        {
+            "info": [{"id": i, "question": f"q{i}"} for i in range(n)],
+            "answer": ["" for _ in range(n)],
+        }
+    )
+
+
+# ── SandboxSpec ─────────────────────────────────────────────────────────
+
+
+def test_sandbox_spec_defaults():
+    spec = SandboxSpec()
+    assert spec.image == "python:3.11-slim"
+    assert spec.cpu_cores == 4
+
+
+def test_sandbox_spec_custom():
+    spec = SandboxSpec(image="lean-tactic:v4.27", gpu_count=1)
+    assert spec.image == "lean-tactic:v4.27"
+    assert spec.gpu_count == 1
+
+
+# ── Task from SandboxTaskSet ───────────────────────────────────────────
+
+
+def test_task_sandbox_spec():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert isinstance(task, Task)
+    assert task.sandbox_spec is not None
+    assert task.sandbox_spec.image == "python:3.11-slim"
+    assert task.sandbox_spec.cpu_cores == 2
+
+
+def test_task_image():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert task.image == "python:3.11-slim"
+
+
+def test_task_workdir():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert task.workdir == "/testbed"
+
+
+def test_task_repr_sandbox():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert "python:3.11-slim" in repr(task)
+
+
+# ── Task from plain TaskSet ────────────────────────────────────────────
+
+
+def test_task_no_sandbox():
+    ts = MockTaskSet(dataset=_make_dataset(), name="math")
+    task = ts[0]
+    assert task.sandbox_spec is None
+    assert task.image is None
+
+
+def test_task_repr_no_sandbox():
+    ts = MockTaskSet(dataset=_make_dataset(), name="math")
+    task = ts[0]
+    assert "no sandbox" in repr(task)
+
+
+# ── TaskSet ─────────────────────────────────────────────────────────────
+
+
+def test_taskset_isinstance():
+    ts = MockTaskSet(dataset=_make_dataset(), name="math")
+    assert not isinstance(ts, SandboxTaskSet)
+
+    ts2 = MockSandboxTaskSet(dataset=_make_dataset(), name="swe")
+    assert isinstance(ts2, SandboxTaskSet)
+
+
+def test_taskset_len():
+    ts = MockTaskSet(dataset=_make_dataset(5), name="test")
+    assert len(ts) == 5
+
+
+def test_taskset_iter():
+    ts = MockTaskSet(dataset=_make_dataset(3), name="test")
+    tasks = list(ts)
+    assert len(tasks) == 3
+    assert all(isinstance(t, Task) for t in tasks)
+
+
+def test_taskset_filter():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(5), name="test")
+    filtered = ts.filter(lambda ex: ex["info"]["id"] < 3)
+    assert len(filtered) == 3
+    assert isinstance(filtered, MockSandboxTaskSet)
+
+
+def test_taskset_take():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(5), name="test")
+    taken = ts.take(2)
+    assert len(taken) == 2
+    assert isinstance(taken, MockSandboxTaskSet)
+
+
+def test_taskset_repr():
+    ts = MockTaskSet(dataset=_make_dataset(), name="mytest")
+    assert "mytest" in repr(ts)
+    assert "3" in repr(ts)
+
+
+@pytest.mark.asyncio
+async def test_composable_env_exports_task_workdir():
+    taskset = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(run_command="true"),
+    )
+
+    env_vars = await env.build_env_vars(
+        {
+            "info": {"id": 0},
+            "interception_base_url": "https://test.trycloudflare.com/v1",
+        }
+    )
+
+    assert env_vars["AGENT_WORKDIR"] == "/testbed"
+    assert env_vars["FOO"] == "bar"
+
+
+@pytest.mark.asyncio
+async def test_composable_env_quotes_paths_in_mkdir_command():
+    taskset = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(
+            run_command="true",
+            instruction_path="/tmp/with space/prompt.txt",
+            system_prompt="system",
+            system_prompt_path="/tmp/other path/system.txt",
+        ),
+    )
+    env.sandbox_client = SimpleNamespace(
+        execute_command=AsyncMock(),
+        teardown=lambda: None,
+    )
+    env.taskset.setup = AsyncMock()
+    env.upload_content = AsyncMock()
+
+    await env.post_sandbox_setup({"sandbox_id": "sbx", "info": {"id": 0}})
+
+    env.sandbox_client.execute_command.assert_awaited_once_with(
+        "sbx",
+        "mkdir -p '/tmp/other path' '/tmp/with space'",
+        timeout=10,
+    )
+
+
+@pytest.mark.asyncio
+async def test_composable_env_quotes_log_path_when_collecting_logs():
+    taskset = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(
+            run_command="true",
+            log_path="/tmp/log dir/agent.log",
+        ),
+    )
+    env.sandbox_client = SimpleNamespace(
+        execute_command=AsyncMock(
+            return_value=SimpleNamespace(stdout="agent log\n", stderr="", exit_code=0)
+        ),
+        teardown=lambda: None,
+    )
+
+    state = {"sandbox_id": "sbx", "timing": {"total_ms": 0}}
+
+    await env.post_rollout(state)
+
+    env.sandbox_client.execute_command.assert_awaited_once_with(
+        "sbx",
+        "cat '/tmp/log dir/agent.log' 2>/dev/null || echo '<no logs>'",
+        working_dir=None,
+    )
+    assert state["agent_logs"] == "agent log"

{verifiers-0.1.12.dev1 → verifiers-0.1.12.dev3}/tests/test_endpoint_registry.py

@@ -1,5 +1,9 @@
 from pathlib import Path
 
+import pytest
+from pydantic import ValidationError
+
+from verifiers.types import ClientConfig
 from verifiers.utils.eval_utils import load_endpoints
 
 
@@ -220,3 +224,96 @@ def test_load_endpoints_toml_accepts_type_shorthand(tmp_path: Path):
     endpoints = load_endpoints(str(registry_path))
 
     assert endpoints["haiku"][0]["api_client_type"] == "anthropic_messages"
+
+
+def test_load_endpoints_toml_accepts_headers_table(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "proxy"\n'
+        'model = "m"\n'
+        'url = "https://api.example/v1"\n'
+        'key = "K"\n'
+        'headers = { "X-Custom" = "v1" }\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints["proxy"][0]["extra_headers"] == {"X-Custom": "v1"}
+
+
+def test_load_endpoints_toml_accepts_extra_headers_alias(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "proxy"\n'
+        'model = "m"\n'
+        'url = "https://api.example/v1"\n'
+        'key = "K"\n'
+        'extra_headers = { "X-A" = "a" }\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints["proxy"][0]["extra_headers"] == {"X-A": "a"}
+
+
+def test_load_endpoints_toml_rejects_headers_and_extra_headers_together(
+    tmp_path: Path,
+):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "proxy"\n'
+        'model = "m"\n'
+        'url = "https://api.example/v1"\n'
+        'key = "K"\n'
+        'headers = { "X-A" = "a" }\n'
+        'extra_headers = { "X-B" = "b" }\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints == {}
+
+
+def test_load_endpoints_python_registry_accepts_headers_dict(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.py"
+    registry_path.write_text(
+        "ENDPOINTS = {\n"
+        '    "p": {"model": "m", "url": "https://x/v1", "key": "K", '
+        '"headers": {"X-Foo": "bar"}},\n'
+        "}\n",
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints["p"][0]["extra_headers"] == {"X-Foo": "bar"}
+
+
+def test_load_endpoints_malformed_headers_string_falls_back_to_empty_registry(
+    tmp_path: Path,
+):
+    toml_path = tmp_path / "endpoints.toml"
+    toml_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "x"\n'
+        'model = "m"\n'
+        'url = "https://api.example/v1"\n'
+        'key = "K"\n'
+        'headers = "invalid"\n',
+        encoding="utf-8",
+    )
+
+    assert load_endpoints(str(toml_path)) == {}
+
+
+def test_client_config_validates_extra_header_keys():
+    with pytest.raises(ValidationError):
+        ClientConfig(extra_headers={"": "x"})
+    with pytest.raises(ValidationError):
+        ClientConfig(extra_headers={"X": 1})  # type: ignore[arg-type]

{verifiers-0.1.12.dev1 → verifiers-0.1.12.dev3}/tests/test_eval_cli.py

@@ -40,6 +40,7 @@ def run_cli(make_metadata, make_state, make_input):
             "api_key_var": "OPENAI_API_KEY",
             "api_base_url": "https://api.openai.com/v1",
             "header": None,
+            "headers": None,
             "num_examples": 1,
             "rollouts_per_example": 1,
             "max_concurrent": 1,
@@ -229,6 +230,87 @@ def test_cli_temperature_not_added_when_none(monkeypatch, run_cli):
     assert "temperature" not in sa
 
 
+def test_cli_headers_table_and_list_merge(monkeypatch, run_cli):
+    captured = run_cli(
+        monkeypatch,
+        {
+            "headers": {"X-A": "a", "X-B": "b"},
+            "header": ["X-B: override", "X-C: c"],
+        },
+        endpoints={},
+    )
+
+    assert captured["configs"][0].client_config.extra_headers == {
+        "X-A": "a",
+        "X-B": "override",
+        "X-C": "c",
+    }
+
+
+def test_cli_registry_headers_merged_with_eval_toml(tmp_path, monkeypatch, run_cli):
+    cfg = tmp_path / "eval.toml"
+    cfg.write_text(
+        "[[eval]]\n"
+        'env_id = "env1"\n'
+        'model = "gpt-5-mini"\n'
+        'headers = { "X-Table" = "t" }\n'
+        'header = [ "X-List: l", "X-Table: override" ]\n',
+        encoding="utf-8",
+    )
+    captured = run_cli(
+        monkeypatch,
+        {"env_id_or_config": str(cfg)},
+        endpoints={
+            "gpt-5-mini": [
+                {
+                    "model": "gpt-5-mini",
+                    "url": "https://a.example/v1",
+                    "key": "OPENAI_API_KEY",
+                    "extra_headers": {"X-Reg": "r"},
+                }
+            ]
+        },
+    )
+
+    assert captured["configs"][0].client_config.extra_headers == {
+        "X-Reg": "r",
+        "X-Table": "override",
+        "X-List": "l",
+    }
+
+
+def test_cli_multi_variant_preserves_per_row_registry_headers(monkeypatch, run_cli):
+    captured = run_cli(
+        monkeypatch,
+        {
+            "model": "gpt-5-mini",
+            "api_key_var": None,
+            "api_base_url": None,
+            "header": ["X-Eval: e"],
+        },
+        endpoints={
+            "gpt-5-mini": [
+                {
+                    "model": "gpt-5-mini",
+                    "url": "https://a.example/v1",
+                    "key": "OPENAI_API_KEY",
+                    "extra_headers": {"X-Row": "a"},
+                },
+                {
+                    "model": "gpt-5-mini",
+                    "url": "https://b.example/v1",
+                    "key": "OPENAI_API_KEY",
+                    "extra_headers": {"X-Row": "b"},
+                },
+            ]
+        },
+    )
+
+    cfgs = captured["configs"][0].client_config.endpoint_configs
+    assert cfgs[0].extra_headers == {"X-Row": "a", "X-Eval": "e"}
+    assert cfgs[1].extra_headers == {"X-Row": "b", "X-Eval": "e"}
+
+
 def test_cli_endpoint_alias_multi_variant_sets_multi_base_urls(monkeypatch, run_cli):
     captured = run_cli(
         monkeypatch,

{verifiers-0.1.12.dev1 → verifiers-0.1.12.dev3}/tests/test_gepa_cli.py

@@ -3,7 +3,33 @@ from pathlib import Path
 
 import pytest
 
-from verifiers.scripts.gepa import load_gepa_toml_config, resolve_gepa_config_args
+from verifiers.scripts.gepa import (
+    _gepa_extra_headers_from_group,
+    load_gepa_toml_config,
+    resolve_gepa_config_args,
+)
+
+
+def test_gepa_extra_headers_from_group_requires_consistent_variants():
+    with pytest.raises(ValueError, match="different headers"):
+        _gepa_extra_headers_from_group(
+            [
+                {"extra_headers": {"X-A": "1"}},
+                {"extra_headers": {"X-A": "2"}},
+            ],
+            "my-alias",
+        )
+
+
+def test_gepa_extra_headers_from_group_returns_first_row_dict():
+    h = _gepa_extra_headers_from_group(
+        [
+            {"extra_headers": {"X-A": "x"}},
+            {"extra_headers": {"X-A": "x"}},
+        ],
+        "my-alias",
+    )
+    assert h == {"X-A": "x"}
 
 
 def test_load_gepa_toml_config_reads_env_table(tmp_path: Path):