verifiers 0.1.15.dev18__tar.gz → 0.1.15.dev151__tar.gz

{verifiers-0.1.15.dev18 → verifiers-0.1.15.dev151}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev18
+Version: 0.1.15.dev151
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev18 → verifiers-0.1.15.dev151}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [project]
@@ -7,7 +7,7 @@ name = "verifiers"
 authors = [
     {name = "William Brown", email = "williambrown97@gmail.com"},
 ]
-dynamic = ["version"]  # load from verifiers/__init__.py
+dynamic = ["version"]  # derived from git tags via hatch-vcs
 description = "Verifiers: Environments for LLM Reinforcement Learning"
 readme = "README.md"
 license = "MIT"
@@ -220,7 +220,20 @@ vf-vllm = "verifiers.scripts.vllm:main"
 
 # hatchling configuration
 [tool.hatch.version]
-path = "verifiers/__init__.py"
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version"
+git_describe_command = [
+    "git",
+    "describe",
+    "--tags",
+    "--long",
+    "--match",
+    "v[0-9]*.[0-9]*.[0-9]*",
+    "--exclude",
+    "*dev*",
+]
 
 [tool.hatch.build.targets.sdist]
 include = [

{verifiers-0.1.15.dev18 → verifiers-0.1.15.dev151}/tests/test_v1_config_extension.py

@@ -28,6 +28,7 @@ from verifiers.v1.utils.config_utils import coerce_config, explicit_config_data
 
 
 REF_MODULE = "v1_config_extension_refs"
+EVAL_FALLBACK_WARNING = "eval_dataset is not set, falling back to train dataset"
 
 
 def load_tasks(split: vf.TaskSplit = "train") -> list[dict[str, object]]:
@@ -782,13 +783,107 @@ async def test_sandbox_program_ref_uses_config_module(
     assert captured["fn_ref"] == f"{__name__}:program_fn"
 
 
-def test_taskset_get_eval_dataset_uses_eval_split() -> None:
+def test_taskset_get_eval_dataset_uses_load_tasks_eval_split() -> None:
     taskset = make_taskset()
 
     assert taskset.get_dataset()[0]["answer"] == "ok"
     assert taskset.get_eval_dataset()[0]["answer"] == "eval ok"
 
 
+def test_taskset_load_tasks_accepts_train_and_eval_splits() -> None:
+    class SplitTaskset(Taskset):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            answer = "eval" if split == "eval" else "train"
+            return [{"prompt": [], "answer": answer}]
+
+    taskset = SplitTaskset()
+
+    assert taskset.load_tasks(split="train")[0]["answer"] == "train"
+    assert taskset.load_tasks(split="eval")[0]["answer"] == "eval"
+
+
+def test_taskset_get_eval_dataset_returns_empty_dataset_for_empty_eval_split() -> None:
+    class TrainOnlyTaskset(Taskset):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            if split == "eval":
+                return []
+            return [{"prompt": [], "answer": "train"}]
+
+    taskset = TrainOnlyTaskset()
+
+    assert taskset.get_dataset()[0]["answer"] == "train"
+    assert len(taskset.get_eval_dataset()) == 0
+
+
+def test_empty_eval_split_uses_environment_train_fallback(monkeypatch) -> None:
+    class EmptyEvalTaskset(Taskset):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            if split == "eval":
+                return []
+            return [{"prompt": [], "answer": "train"}]
+
+    env = Env(taskset=EmptyEvalTaskset())
+    warnings: list[str] = []
+    monkeypatch.setattr(env.logger, "warning", warnings.append)
+
+    assert len(env.taskset.get_eval_dataset()) == 0
+    eval_rows = env.get_eval_dataset()
+
+    assert eval_rows[0]["answer"] == "train"
+    assert warnings == [EVAL_FALLBACK_WARNING]
+
+
+def test_empty_train_split_uses_environment_eval_only_dataset() -> None:
+    class EmptyTrainTaskset(Taskset):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            if split == "train":
+                return []
+            return [{"prompt": [], "answer": "eval"}]
+
+    env = Env(taskset=EmptyTrainTaskset())
+
+    assert env.get_eval_dataset()[0]["answer"] == "eval"
+    with pytest.raises(ValueError, match="dataset is not set"):
+        env.get_dataset()
+
+
+def test_empty_train_and_eval_splits_match_environment_missing_dataset_error(
+    monkeypatch,
+) -> None:
+    env = Env(taskset=Taskset())
+    warnings: list[str] = []
+    monkeypatch.setattr(env.logger, "warning", warnings.append)
+
+    with pytest.raises(ValueError, match="dataset is not set"):
+        env.get_dataset()
+    with pytest.raises(ValueError, match="dataset is not set"):
+        env.get_eval_dataset()
+    assert warnings == [EVAL_FALLBACK_WARNING]
+
+
+def test_empty_taskset_splits_are_checked_once_by_environment() -> None:
+    calls = {"train": 0, "eval": 0}
+
+    class EmptyTaskset(Taskset):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            if split == "train":
+                calls["train"] += 1
+            else:
+                calls["eval"] += 1
+            return []
+
+    env = Env(taskset=EmptyTaskset())
+
+    for _ in range(2):
+        with pytest.raises(ValueError, match="dataset is not set"):
+            env.get_dataset()
+    for _ in range(2):
+        with pytest.raises(ValueError, match="dataset is not set"):
+            env.get_eval_dataset()
+
+    assert calls == {"train": 1, "eval": 1}
+
+
 def test_env_passes_taskset_eval_dataset_to_environment() -> None:
     env = Env(
         taskset=make_taskset(),
@@ -2382,7 +2477,6 @@ def test_config_schema_is_visible_from_primary_types() -> None:
     assert "- toolsets:" in taskset_schema
     assert "- toolsets:" in HarnessConfig.schema_text()
     assert "- tasks:" not in taskset_schema
-    assert "- eval_tasks:" not in taskset_schema
     assert "- program:" in HarnessConfig.schema_text()
     assert "- image:" in vf.SandboxConfig.schema_text()
     assert "- bindings:" in vf.ToolsetConfig.schema_text()
@@ -3406,6 +3500,10 @@ def test_nested_configs_validate_and_feed_runtime_objects() -> None:
         packages="numpy",
         setup_commands="echo ready",
         scope="group",
+        create_concurrency=7,
+        create_rate_per_second=1.5,
+        delete_concurrency=3,
+        delete_rate_per_second=2.5,
     )
     harness = make_harness(program={"sandbox": True}, sandbox=sandbox)
 
@@ -3414,6 +3512,10 @@ def test_nested_configs_validate_and_feed_runtime_objects() -> None:
     assert harness.sandbox.packages == ["numpy"]
     assert harness.sandbox.setup_commands == ["echo ready"]
     assert harness.sandbox.scope == "group"
+    assert harness.sandbox.create_concurrency == 7
+    assert harness.sandbox.create_rate_per_second == 1.5
+    assert harness.sandbox.delete_concurrency == 3
+    assert harness.sandbox.delete_rate_per_second == 2.5
 
     toolset = Toolset(
         config=vf.ToolsetConfig(
@@ -3435,6 +3537,9 @@ def test_nested_configs_reject_unknown_fields() -> None:
     with pytest.raises(ValueError):
         vf.SandboxConfig.model_validate({"image": "python:3.11", "unknown": True})
 
+    with pytest.raises(ValueError):
+        HarnessConfig.model_validate({"sandbox_create_concurrency": 2})
+
     with pytest.raises(ValueError):
         vf.ToolsetConfig.model_validate({"tools": [], "show": ["a"], "hide": ["b"]})
 

{verifiers-0.1.15.dev18 → verifiers-0.1.15.dev151}/tests/test_v1_openreward_taskset.py

@@ -59,10 +59,23 @@ class FakeOpenRewardSession:
         )
 
 
+class FakeOpenRewardSplit:
+    def __init__(self, name: str, type: str):
+        self.name = name
+        self.type = type
+
+
 class FakeOpenRewardEnvironment:
     def __init__(self):
         self.sessions: list[FakeOpenRewardSession] = []
         self.task_range_calls: list[tuple[str, int | None, int | None]] = []
+        self.splits = [
+            FakeOpenRewardSplit(name="train", type="train"),
+            FakeOpenRewardSplit(name="official-test", type="test"),
+        ]
+
+    def list_splits(self) -> list[FakeOpenRewardSplit]:
+        return self.splits
 
     def list_tasks(self, split: str) -> list[OpenRewardTask]:
         return [
@@ -162,6 +175,37 @@ def test_openreward_taskset_loads_serializable_tasks(fake_openreward_client):
     assert set(taskset.named_toolsets) == {"openreward"}
 
 
+def test_openreward_taskset_load_tasks_eval_uses_test_split(fake_openreward_client):
+    taskset = openreward.OpenRewardTaskset(
+        config=openreward.OpenRewardTasksetConfig(
+            environment="owner/env",
+            split="train",
+            num_eval_examples=2,
+        )
+    )
+
+    tasks = list(taskset.load_tasks(split="eval"))
+
+    assert fake_openreward_client.task_range_calls == [("official-test", 0, 2)]
+    assert tasks[0]["openreward"]["split"] == "official-test"
+    assert tasks[0]["openreward"]["task"]["task_spec"] == {"id": "official-test-0"}
+
+
+def test_openreward_taskset_eval_split_empty_without_test_split(
+    fake_openreward_client,
+):
+    fake_openreward_client.splits = [FakeOpenRewardSplit(name="train", type="train")]
+    taskset = openreward.OpenRewardTaskset(
+        config=openreward.OpenRewardTasksetConfig(
+            environment="owner/env",
+            num_eval_examples=2,
+        )
+    )
+
+    assert len(taskset.get_eval_dataset()) == 0
+    assert fake_openreward_client.task_range_calls == []
+
+
 @pytest.mark.asyncio
 async def test_openreward_taskset_setup_and_tool_call(fake_openreward_client):
     taskset = openreward.OpenRewardTaskset(

{verifiers-0.1.15.dev18 → verifiers-0.1.15.dev151}/tests/test_v1_rlm_swe.py

@@ -82,7 +82,6 @@ def test_rlm_harness_accepts_typed_config_surface():
             program=RLMProgramConfig(
                 local_checkout="/tmp/checkout",
                 rlm_tools=["bash", "edit"],
-                rlm_max_turns=7,
                 rlm_exec_timeout=11,
                 env_vars={"CUSTOM": "1"},
             )
@@ -93,7 +92,6 @@ def test_rlm_harness_accepts_typed_config_surface():
 
     assert harness.config.program.rlm_tools == ["bash", "edit"]
     assert program_env["RLM_TOOLS"] == "bash,edit"
-    assert program_env["RLM_MAX_TURNS"] == "7"
     assert program_env["RLM_EXEC_TIMEOUT"] == "11"
     assert program_env["CUSTOM"] == "1"