verifiers 0.1.15.dev158__tar.gz → 0.1.15.dev159__tar.gz

{verifiers-0.1.15.dev158 → verifiers-0.1.15.dev159}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev158
+Version: 0.1.15.dev159
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -54,17 +54,17 @@ Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.0.0; extra == 'browser'
 Provides-Extra: harnesses
-Requires-Dist: harnesses>=0.1.1; extra == 'harnesses'
+Requires-Dist: harnesses>=0.1.2; extra == 'harnesses'
 Provides-Extra: nemogym
-Requires-Dist: harnesses[nemogym]>=0.1.1; extra == 'nemogym'
-Requires-Dist: tasksets[nemogym]>=0.1.1; extra == 'nemogym'
+Requires-Dist: harnesses[nemogym]>=0.1.2; extra == 'nemogym'
+Requires-Dist: tasksets[nemogym]>=0.1.5; extra == 'nemogym'
 Provides-Extra: openenv
-Requires-Dist: tasksets[openenv]>=0.1.1; extra == 'openenv'
+Requires-Dist: tasksets[openenv]>=0.1.5; extra == 'openenv'
 Provides-Extra: openreward
-Requires-Dist: tasksets[openreward]>=0.1.1; extra == 'openreward'
+Requires-Dist: tasksets[openreward]>=0.1.5; extra == 'openreward'
 Provides-Extra: packages
-Requires-Dist: harnesses>=0.1.1; extra == 'packages'
-Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.1; extra == 'packages'
+Requires-Dist: harnesses>=0.1.2; extra == 'packages'
+Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.5; extra == 'packages'
 Provides-Extra: renderers
 Requires-Dist: renderers>=0.1.8.dev28; extra == 'renderers'
 Provides-Extra: rg
@@ -84,7 +84,7 @@ Provides-Extra: ta
 Requires-Dist: nltk; extra == 'ta'
 Requires-Dist: textarena; extra == 'ta'
 Provides-Extra: tasksets
-Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.1; extra == 'tasksets'
+Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.5; extra == 'tasksets'
 Description-Content-Type: text/markdown
 
 <p align="center">

{verifiers-0.1.15.dev158 → verifiers-0.1.15.dev159}/pyproject.toml

@@ -89,14 +89,14 @@ ta = [
     "textarena",
 ]
 tasksets = [
-    "tasksets[openenv,openreward,ta]>=0.1.1",
+    "tasksets[openenv,openreward,ta]>=0.1.5",
 ]
 harnesses = [
-    "harnesses>=0.1.1",
+    "harnesses>=0.1.2",
 ]
 packages = [
-    "harnesses>=0.1.1",
-    "tasksets[openenv,openreward,ta]>=0.1.1",
+    "harnesses>=0.1.2",
+    "tasksets[openenv,openreward,ta]>=0.1.5",
 ]
 browser = [
     "stagehand>=3.0.0",
@@ -104,14 +104,14 @@ browser = [
     "python-dotenv>=1.0.0",
 ]
 openenv = [
-    "tasksets[openenv]>=0.1.1",
+    "tasksets[openenv]>=0.1.5",
 ]
 openreward = [
-    "tasksets[openreward]>=0.1.1",
+    "tasksets[openreward]>=0.1.5",
 ]
 nemogym = [
-    "harnesses[nemogym]>=0.1.1",
-    "tasksets[nemogym]>=0.1.1",
+    "harnesses[nemogym]>=0.1.2",
+    "tasksets[nemogym]>=0.1.5",
 ]
 renderers = [
     "renderers>=0.1.8.dev28",

{verifiers-0.1.15.dev158 → verifiers-0.1.15.dev159}/tests/test_imports.py

@@ -17,6 +17,9 @@ PACKAGE_SYMBOLS = {
     "OpenCodeConfig",
     "Pi",
     "PiConfig",
+    "ReplayHarness",
+    "ReplayTaskset",
+    "ReplayTasksetConfig",
     "RLM",
     "RLMConfig",
     "Terminus2",

{verifiers-0.1.15.dev158 → verifiers-0.1.15.dev159}/tests/test_init_script.py

@@ -106,7 +106,7 @@ def test_init_openenv_writes_v1_taskset_template(tmp_path: Path) -> None:
     assert "taskset=vf.load_taskset(config=config.taskset)" in content
     assert "harness=vf.load_harness(config=config.harness)" in content
     assert "vf.OpenEnvEnv" not in content
-    assert '"tasksets[openenv]>=0.1.1"' in pyproject
+    assert '"tasksets[openenv]>=0.1.5"' in pyproject
 
 
 def test_init_openenv_multifile_exports_taskset_loader(tmp_path: Path) -> None:

{verifiers-0.1.15.dev158 → verifiers-0.1.15.dev159}/tests/test_v1_config_extension.py

@@ -2738,6 +2738,68 @@ def test_load_environment_validates_typed_env_config_arg(
     }
 
 
+def test_load_environment_accepts_top_level_taskset_harness_shorthand(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "typed_env_config_shorthand"
+    module = types.ModuleType(module_name)
+    seen: dict[str, object] = {}
+
+    def load_environment(split: str = "train", *, config: EnvConfig) -> Env:
+        seen["split"] = split
+        seen["config"] = config
+        return Env(
+            taskset=make_taskset(config=config.taskset),
+            harness=make_harness(config=config.harness),
+        )
+
+    module.load_environment = load_environment
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    env = vf.load_environment(
+        "typed-env-config-shorthand",
+        split="test",
+        taskset={"taskset_id": "top-level"},
+        harness={"model": {"name": "top-level-model"}},
+    )
+
+    assert seen["split"] == "test"
+    config = seen["config"]
+    assert isinstance(config, EnvConfig)
+    assert config.taskset.taskset_id == "top-level"
+    assert config.harness.model.name == "top-level-model"
+    assert env.taskset.config.taskset_id == "top-level"
+    assert env.harness.config.model.name == "top-level-model"
+    assert env.env_args == {
+        "split": "test",
+        "taskset": {"taskset_id": "top-level"},
+        "harness": {"model": {"name": "top-level-model"}},
+    }
+
+
+def test_load_environment_rejects_duplicate_top_level_config_shorthand(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "typed_env_config_duplicate_shorthand"
+    module = types.ModuleType(module_name)
+
+    def load_environment(config: EnvConfig) -> Env:
+        return Env(
+            taskset=make_taskset(config=config.taskset),
+            harness=make_harness(config=config.harness),
+        )
+
+    module.load_environment = load_environment
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    with pytest.raises(RuntimeError, match="not both"):
+        vf.load_environment(
+            "typed-env-config-duplicate-shorthand",
+            config={"taskset": {"taskset_id": "nested"}},
+            taskset={"taskset_id": "top-level"},
+        )
+
+
 def test_load_environment_validates_env_config_subclass_sections(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -3099,6 +3161,36 @@ def test_load_environment_composes_component_package_without_root_loader(
     assert env.harness.config.max_turns == 3
 
 
+def test_load_environment_component_package_accepts_top_level_config_shorthand(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "component_only_taskset_shorthand"
+    module = types.ModuleType(module_name)
+
+    class LocalTasksetConfig(TasksetConfig):
+        answer: str = "configured"
+
+    class LocalTaskset(Taskset[LocalTasksetConfig]):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            return [{"prompt": [], "answer": f"{split}:{self.config.answer}"}]
+
+    def load_taskset(config: LocalTasksetConfig) -> LocalTaskset:
+        return LocalTaskset(config=config)
+
+    module.load_taskset = load_taskset
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    env = vf.load_environment(
+        "component-only-taskset-shorthand",
+        taskset={"answer": "top-level"},
+        harness={"max_turns": 2},
+    )
+
+    assert env.taskset.get_dataset()[0]["answer"] == "train:top-level"
+    assert type(env.harness) is Harness
+    assert env.harness.config.max_turns == 2
+
+
 def test_load_environment_delegates_missing_child_loaders_by_config_id(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

verifiers-0.1.15.dev159/tests/test_v1_replay_harness.py

@@ -0,0 +1,327 @@
+import json
+from pathlib import Path
+
+import pytest
+
+import verifiers as vf
+from harnesses import ReplayHarness
+from tasksets import ReplayTaskset, ReplayTasksetConfig
+from tasksets.replay import replay_task_record
+
+
+class NoModelClient:
+    def __init__(self) -> None:
+        self.requests = 0
+
+    async def get_response(self, **kwargs: object) -> object:
+        _ = kwargs
+        self.requests += 1
+        raise AssertionError("ReplayHarness must not request model completions.")
+
+
+class InlineReplayTaskset(ReplayTaskset):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+        if split == "eval":
+            return []
+        return [
+            {
+                "messages": [
+                    {"role": "user", "content": "Reverse abc."},
+                    {"role": "assistant", "content": "cba"},
+                    {"role": "user", "content": "Now uppercase it."},
+                    {
+                        "role": "assistant",
+                        "content": "CBA",
+                        "reasoning_content": "uppercased the prior answer",
+                    },
+                    {"role": "user", "content": "Thanks."},
+                ],
+            }
+        ]
+
+
+class ManyTurnReplayTaskset(ReplayTaskset):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+        if split == "eval":
+            return []
+        messages = []
+        for index in range(11):
+            messages.append({"role": "user", "content": f"Turn {index}?"})
+            messages.append({"role": "assistant", "content": f"reply {index}"})
+        return [{"messages": messages}]
+
+
+@pytest.mark.asyncio
+async def test_replay_harness_prints_assistant_messages_into_trajectory() -> None:
+    env = vf.Env(
+        taskset=InlineReplayTaskset(),
+        harness=ReplayHarness(config=vf.HarnessConfig()),
+    )
+    client = NoModelClient()
+
+    state = await env.rollout(
+        dict(env.get_dataset()[0]),
+        client=client,
+        model="mock-model",
+    )
+
+    assert client.requests == 0
+    assert state["stop_condition"] == "replayed_messages"
+    assert state["num_model_requests"] == 2
+    assert state["prompt"] == [{"role": "user", "content": "Reverse abc."}]
+    assert state["completion"] == [
+        {"role": "assistant", "content": "cba"},
+        {"role": "user", "content": "Now uppercase it."},
+        {
+            "role": "assistant",
+            "content": "CBA",
+            "reasoning_content": "uppercased the prior answer",
+        },
+    ]
+    assert state["completion"][-1]["role"] == "assistant"
+
+    first, second = state["trajectory"]
+    assert first["prompt"] == [{"role": "user", "content": "Reverse abc."}]
+    assert first["completion"] == [{"role": "assistant", "content": "cba"}]
+    assert first["tokens"] is None
+    assert "tokens" not in first["response"]["message"]
+
+    assert second["prompt"] == [
+        {"role": "user", "content": "Reverse abc."},
+        {"role": "assistant", "content": "cba"},
+        {"role": "user", "content": "Now uppercase it."},
+    ]
+    assert second["completion"] == [
+        {
+            "role": "assistant",
+            "content": "CBA",
+            "reasoning_content": "uppercased the prior answer",
+        }
+    ]
+    assert second["tokens"] is None
+    assert "tokens" not in second["response"]["message"]
+
+
+@pytest.mark.asyncio
+async def test_replay_harness_marks_partial_replay_as_truncated() -> None:
+    env = vf.Env(
+        taskset=InlineReplayTaskset(),
+        harness=ReplayHarness(config=vf.HarnessConfig(max_turns=1)),
+    )
+
+    state = await env.rollout(
+        dict(env.get_dataset()[0]),
+        client=NoModelClient(),
+        model="mock-model",
+    )
+
+    assert state["stop_condition"] == "max_turns_reached"
+    assert state["is_truncated"] is True
+    assert state["num_model_requests"] == 1
+    assert state["completion"] == [{"role": "assistant", "content": "cba"}]
+    step = state["trajectory"][0]
+    assert step["is_truncated"] is True
+    assert step["response"]["message"]["is_truncated"] is True
+
+
+@pytest.mark.asyncio
+async def test_replay_harness_defaults_to_all_assistant_messages() -> None:
+    assert vf.HarnessConfig().max_turns == -1
+    env = vf.Env(
+        taskset=ManyTurnReplayTaskset(),
+        harness=ReplayHarness(config=vf.HarnessConfig()),
+    )
+
+    state = await env.rollout(
+        dict(env.get_dataset()[0]),
+        client=NoModelClient(),
+        model="mock-model",
+    )
+
+    assert state["stop_condition"] == "replayed_messages"
+    assert state["is_truncated"] is False
+    assert state["num_model_requests"] == 11
+    assert len(state["trajectory"]) == 11
+    assert state["completion"][-1] == {"role": "assistant", "content": "reply 10"}
+
+
+def test_replay_taskset_loads_configured_local_jsonl_data(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "examples.jsonl").write_text(
+        "\n".join(
+            [
+                json.dumps(
+                    {
+                        "messages": [
+                            {"role": "user", "content": "Say ok."},
+                            {"role": "assistant", "content": "ok"},
+                        ]
+                    }
+                ),
+                json.dumps(
+                    {
+                        "messages": [
+                            {"role": "user", "content": "Say yes."},
+                            {"role": "assistant", "content": "yes"},
+                        ]
+                    }
+                ),
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    assert taskset.load_tasks() == [
+        {
+            "messages": [
+                {"role": "user", "content": "Say ok."},
+                {"role": "assistant", "content": "ok"},
+            ]
+        },
+        {
+            "messages": [
+                {"role": "user", "content": "Say yes."},
+                {"role": "assistant", "content": "yes"},
+            ]
+        },
+    ]
+
+
+def test_replay_taskset_loads_subclass_local_jsonl_data(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "example.jsonl").write_text(
+        json.dumps(
+            {
+                "messages": [
+                    {"role": "user", "content": "Say ok."},
+                    {"role": "assistant", "content": "ok"},
+                ]
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    local_taskset_type = type(
+        "LocalReplayTaskset",
+        (ReplayTaskset,),
+        {"data_dir": str(data_dir)},
+    )
+    taskset = local_taskset_type(config=ReplayTasksetConfig())
+
+    assert taskset.load_tasks() == [
+        {
+            "messages": [
+                {"role": "user", "content": "Say ok."},
+                {"role": "assistant", "content": "ok"},
+            ]
+        }
+    ]
+
+
+def test_replay_taskset_rejects_missing_local_source() -> None:
+    taskset = ReplayTaskset(config=ReplayTasksetConfig())
+
+    with pytest.raises(FileNotFoundError, match="requires dataset or data_dir"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_conflicting_sources(tmp_path: Path) -> None:
+    taskset = ReplayTaskset(
+        config=ReplayTasksetConfig(dataset="owner/dataset", data_dir=str(tmp_path))
+    )
+
+    with pytest.raises(ValueError, match="cannot set both dataset and data_dir"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_empty_local_data_dir(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    with pytest.raises(FileNotFoundError, match="must contain at least one JSONL"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_json_files(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "example.json").write_text(
+        json.dumps(
+            {
+                "messages": [
+                    {"role": "user", "content": "Say ok."},
+                    {"role": "assistant", "content": "ok"},
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    with pytest.raises(ValueError, match=r"accepts only \.jsonl files"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_rejects_non_object_jsonl_rows(tmp_path: Path) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(parents=True)
+    (data_dir / "example.jsonl").write_text("[]\n", encoding="utf-8")
+
+    taskset = ReplayTaskset(config=ReplayTasksetConfig(data_dir=str(data_dir)))
+
+    with pytest.raises(TypeError, match="example.jsonl:1 must contain one JSON object"):
+        taskset.load_tasks()
+
+
+def test_replay_taskset_canonicalizes_messages() -> None:
+    task = replay_task_record(
+        {
+            "messages": [
+                {"role": "user", "content": "Use the tool."},
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": "call_1",
+                            "type": "function",
+                            "function": {
+                                "name": "search",
+                                "arguments": {"query": "abc"},
+                            },
+                        }
+                    ],
+                },
+            ]
+        }
+    )
+
+    assert task["messages"] == [
+        {"role": "user", "content": "Use the tool."},
+        {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "name": "search",
+                    "arguments": '{"query": "abc"}',
+                }
+            ],
+        },
+    ]
+
+
+def test_replay_taskset_rejects_invalid_messages() -> None:
+    with pytest.raises(TypeError, match="messages must be a list"):
+        replay_task_record({"messages": "not a transcript"})
+
+    with pytest.raises(ValueError, match="Unknown role"):
+        replay_task_record({"messages": [{"role": "assistantish", "content": "no"}]})

verifiers-0.1.15.dev159/tests/test_v1_taskset_utils.py

@@ -0,0 +1,120 @@
+import json
+import sys
+import types
+
+from datasets import Dataset
+
+from verifiers.v1 import Taskset
+from verifiers.v1.utils.taskset_utils import dataset_from_result, discover_sibling_dir
+
+
+def task_payload(row: dict) -> dict:
+    return json.loads(row["info"]["task"])
+
+
+def test_dataset_from_result_assigns_example_id_to_iterable_records():
+    dataset = dataset_from_result(
+        [
+            {"question": "Reverse abc.", "answer": "cba"},
+            {"question": "Reverse xyz.", "answer": "zyx"},
+        ],
+        "ReverseTextTaskset",
+    )
+
+    rows = list(dataset)
+    payloads = [task_payload(row) for row in rows]
+
+    assert [row["example_id"] for row in rows] == [0, 1]
+    assert [payload["example_id"] for payload in payloads] == [0, 1]
+    assert all(len(payload["task_id"]) == 32 for payload in payloads)
+    assert {payload["task_id"] for payload in payloads}.isdisjoint({"0", "1"})
+
+
+def test_dataset_from_result_overwrites_existing_example_id_column():
+    raw_dataset = Dataset.from_list(
+        [
+            {"question": "Reverse abc.", "answer": "cba", "example_id": None},
+            {"question": "Reverse xyz.", "answer": "zyx", "example_id": 99},
+        ]
+    )
+
+    dataset = dataset_from_result(raw_dataset, "ReverseTextTaskset")
+
+    rows = list(dataset)
+    payloads = [task_payload(row) for row in rows]
+
+    assert [row["example_id"] for row in rows] == [0, 1]
+    assert [payload["example_id"] for payload in payloads] == [0, 1]
+    assert all(len(payload["task_id"]) == 32 for payload in payloads)
+    assert {payload["task_id"] for payload in payloads}.isdisjoint({"0", "1", "99"})
+
+
+def test_discover_sibling_dir_returns_empty_existing_dir(tmp_path, monkeypatch) -> None:
+    env_dir = tmp_path / "empty_data_env"
+    data_dir = env_dir / "data"
+    data_dir.mkdir(parents=True)
+    module_path = env_dir / "empty_data_env.py"
+    module_path.write_text("", encoding="utf-8")
+
+    module = types.ModuleType("empty_data_env")
+    module.__file__ = str(module_path)
+    monkeypatch.setitem(sys.modules, module.__name__, module)
+    taskset_type = type(
+        "EmptyDataTaskset",
+        (),
+        {"__module__": module.__name__},
+    )
+
+    assert discover_sibling_dir(taskset_type, "data") == data_dir
+
+
+def test_discover_sibling_dir_can_require_non_empty(tmp_path, monkeypatch) -> None:
+    env_dir = tmp_path / "empty_skills_env"
+    skills_dir = env_dir / "skills"
+    skills_dir.mkdir(parents=True)
+    module_path = env_dir / "empty_skills_env.py"
+    module_path.write_text("", encoding="utf-8")
+
+    module = types.ModuleType("empty_skills_env")
+    module.__file__ = str(module_path)
+    monkeypatch.setitem(sys.modules, module.__name__, module)
+    taskset_type = type(
+        "EmptySkillsTaskset",
+        (),
+        {"__module__": module.__name__},
+    )
+
+    assert discover_sibling_dir(taskset_type, "skills", require_non_empty=True) is None
+
+    (skills_dir / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
+
+    assert (
+        discover_sibling_dir(taskset_type, "skills", require_non_empty=True)
+        == skills_dir
+    )
+
+
+def test_taskset_skips_empty_skills_dir(tmp_path, monkeypatch) -> None:
+    env_dir = tmp_path / "empty_taskset_skills_env"
+    skills_dir = env_dir / "skills"
+    skills_dir.mkdir(parents=True)
+    module_path = env_dir / "empty_taskset_skills_env.py"
+    module_path.write_text("", encoding="utf-8")
+
+    module = types.ModuleType("empty_taskset_skills_env")
+    module.__file__ = str(module_path)
+    monkeypatch.setitem(sys.modules, module.__name__, module)
+    taskset_type = type(
+        "EmptyTasksetSkills",
+        (Taskset,),
+        {"__module__": module.__name__},
+    )
+    taskset = taskset_type()
+
+    assert taskset.get_skills_dir() is None
+    assert taskset.get_upload_dirs() == {}
+
+    (skills_dir / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
+
+    assert taskset.get_skills_dir() == skills_dir
+    assert taskset.get_upload_dirs() == {"skills": skills_dir}

{verifiers-0.1.15.dev158 → verifiers-0.1.15.dev159}/verifiers/scripts/init.py

@@ -120,7 +120,7 @@ version = "0.1.0"
 requires-python = ">=3.10"
 dependencies = [
     "verifiers>={vf.__version__}",
-    "tasksets[openenv]>=0.1.1",
+    "tasksets[openenv]>=0.1.5",
 ]
 
 [build-system]