verifiers 0.1.15.dev15__tar.gz → 0.1.15.dev16__tar.gz

{verifiers-0.1.15.dev15 → verifiers-0.1.15.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev15
+Version: 0.1.15.dev16
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev15 → verifiers-0.1.15.dev16}/tests/test_init_script.py

@@ -1,6 +1,4 @@
 from pathlib import Path
-
-import pytest
 import verifiers as vf
 from verifiers.scripts.init import init_environment
 
@@ -21,20 +19,23 @@ def test_init_default_writes_v0_stub(tmp_path: Path) -> None:
     assert "EnvTaskset" not in content
 
 
-def test_init_v1_writes_thin_taskset_template(tmp_path: Path) -> None:
+def test_init_v1_writes_taskset_template(tmp_path: Path) -> None:
     init_environment("bar", path=str(tmp_path), v1=True)
     content = read_env_file(tmp_path, "bar")
 
     assert "class BarTasksetConfig(vf.TasksetConfig):" in content
     assert "class BarTaskset(vf.Taskset[BarTasksetConfig]):" in content
+    assert 'system_prompt: vf.SystemPrompt = "Answer exactly."' in content
+    assert '"""Taskset implementation for bar.' in content
+    assert 'def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:' in content
     assert (
-        'system_prompt: vf.SystemPrompt = "Replace this with the system prompt for bar."'
+        '"""Return serializable task records as a list, generator, or Dataset."""'
         in content
     )
-    assert 'def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:' in content
     assert "def load_system_prompt" not in content
     assert "async def correct_answer(self, task: vf.Task, state: vf.State)" in content
     assert "def load_taskset(config: BarTasksetConfig) -> BarTaskset:" in content
+    assert '"""Typed taskset loader used by vf.load_taskset."""' in content
     assert "return BarTaskset(config=config)" in content
     assert "taskset=vf.load_taskset(config=config.taskset)" in content
     assert '"""Loader pattern for all Taskset/Harness environments."""' in content
@@ -53,8 +54,10 @@ def test_init_v1_template_loads_with_vf_load_environment(
 
     env = vf.load_environment("loadable-v1")
 
-    with pytest.raises(RuntimeError, match="Load tasks"):
-        env.get_dataset()
+    dataset = env.get_dataset()
+
+    assert len(dataset) == 1
+    assert dataset[0]["answer"] == "cba"
 
 
 def test_init_v1_with_harness_writes_harness_stub(tmp_path: Path) -> None:

{verifiers-0.1.15.dev15 → verifiers-0.1.15.dev16}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev15"
+__version__ = "0.1.15.dev16"
 
 import importlib
 import os

{verifiers-0.1.15.dev15 → verifiers-0.1.15.dev16}/verifiers/scripts/init.py

@@ -155,67 +155,72 @@ def load_environment(**kwargs) -> vf.Environment:
     raise NotImplementedError("Implement load_environment here.")
 """
 
-V1_ENVIRONMENT_TEMPLATE = """\
+V1_TASKSET_TEMPLATE = """\
 import verifiers as vf
 
 
 class {taskset_config_name}(vf.TasksetConfig):
-    system_prompt: vf.SystemPrompt = "Replace this with the system prompt for {env_id_dash}."
+    \"\"\"User-facing task settings for {env_id_dash}.\"\"\"
+
+    system_prompt: vf.SystemPrompt = "Answer exactly."
 
 
 class {taskset_name}(vf.Taskset[{taskset_config_name}]):
+    \"\"\"Taskset implementation for {env_id_dash}.
+
+    Add task loading, task-owned toolsets, user behavior, lifecycle hooks,
+    metrics, rewards, and advantages on this class.
+    \"\"\"
+
     def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
-        raise NotImplementedError("Load tasks for {env_id_dash}.")
+        \"\"\"Return serializable task records as a list, generator, or Dataset.\"\"\"
+        if split == "eval":
+            return []
+        return [
+            {
+                "prompt": [{"role": "user", "content": "Reverse abc."}],
+                "answer": "cba",
+                "max_turns": 1,
+            }
+        ]
 
     @vf.reward(weight=1.0)
     async def correct_answer(self, task: vf.Task, state: vf.State) -> float:
-        raise NotImplementedError("Score a completed rollout for {env_id_dash}.")
+        \"\"\"Score the final assistant response for one rollout.\"\"\"
+        messages = vf.get_messages(state.get("completion") or [], role="assistant")
+        if not messages:
+            return 0.0
+        response = str(messages[-1].content or "").strip()
+        return float(response == task["answer"])
 
 
 def load_taskset(config: {taskset_config_name}) -> {taskset_name}:
+    \"\"\"Typed taskset loader used by vf.load_taskset.\"\"\"
     return {taskset_name}(config=config)
-
-
-def load_environment(config: vf.EnvConfig) -> vf.Env:
-    \"\"\"Loader pattern for all Taskset/Harness environments.\"\"\"
-    return vf.Env(
-        taskset=vf.load_taskset(config=config.taskset),
-        harness=vf.load_harness(config=config.harness),
-    )
 """
 
-V1_HARNESS_ENVIRONMENT_TEMPLATE = """\
-import verifiers as vf
-
-
-class {taskset_config_name}(vf.TasksetConfig):
-    system_prompt: vf.SystemPrompt = "Replace this with the system prompt for {env_id_dash}."
-
-
-class {taskset_name}(vf.Taskset[{taskset_config_name}]):
-    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
-        raise NotImplementedError("Load tasks for {env_id_dash}.")
-
-    @vf.reward(weight=1.0)
-    async def correct_answer(self, task: vf.Task, state: vf.State) -> float:
-        raise NotImplementedError("Score a completed rollout for {env_id_dash}.")
 
+V1_HARNESS_TEMPLATE = """\
 
 class {harness_config_name}(vf.HarnessConfig):
-    pass
+    \"\"\"Execution settings for {env_id_dash}.\"\"\"
 
 
 class {harness_name}(vf.Harness[{harness_config_name}]):
-    pass
+    \"\"\"Reusable execution behavior for {env_id_dash}.
 
-
-def load_taskset(config: {taskset_config_name}) -> {taskset_name}:
-    return {taskset_name}(config=config)
+    Add harness-owned program, sandbox, endpoint, model, toolset, or lifecycle
+    behavior here when this environment owns a custom execution mechanism.
+    \"\"\"
 
 
 def load_harness(config: {harness_config_name}) -> {harness_name}:
+    \"\"\"Typed harness loader used by vf.load_harness.\"\"\"
     return {harness_name}(config=config)
+"""
+
 
+V1_ENV_LOADER_TEMPLATE = """\
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
     \"\"\"Loader pattern for all Taskset/Harness environments.\"\"\"
@@ -225,6 +230,11 @@ def load_environment(config: vf.EnvConfig) -> vf.Env:
     )
 """
 
+V1_ENVIRONMENT_TEMPLATE = V1_TASKSET_TEMPLATE + V1_ENV_LOADER_TEMPLATE
+V1_HARNESS_ENVIRONMENT_TEMPLATE = (
+    V1_TASKSET_TEMPLATE + V1_HARNESS_TEMPLATE + V1_ENV_LOADER_TEMPLATE
+)
+
 OPENENV_ENVIRONMENT_TEMPLATE = """\
 import verifiers as vf
 from tasksets import OpenEnvTaskset, OpenEnvTasksetConfig