verifiers 0.1.15.dev16__tar.gz → 0.1.15.dev17__tar.gz

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev16
+Version: 0.1.15.dev17
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/tests/test_gepa_cli.py

@@ -189,6 +189,14 @@ def test_load_gepa_toml_config_requires_env_table(tmp_path: Path):
         load_gepa_toml_config(config_path)
 
 
+def test_repo_gepa_example_configs_are_valid():
+    config_paths = sorted(Path("configs/gepa").glob("*.toml"))
+    assert config_paths
+    for config_path in config_paths:
+        loaded = load_gepa_toml_config(config_path)
+        assert loaded["envs"], f"{config_path} should contain at least one [[env]]"
+
+
 def test_resolve_gepa_config_args_supports_plain_env_id():
     args = argparse.Namespace(env_id_or_config="primeintellect/wordle")
 

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/tests/test_save_utils.py

@@ -258,6 +258,13 @@ class TestSavingResults:
         assert result[0].get("foo") == "bar"  # custom field from make_state fixture
         assert result[0]["reward"] == 1.0
 
+    def test_states_to_outputs_requires_example_id(self, make_state):
+        state = make_state()
+        del state["example_id"]
+
+        with pytest.raises(KeyError):
+            states_to_outputs([state], state_columns=[])
+
     def test_states_to_outputs_completion_keeps_messages(self, make_state):
         states = [
             make_state(
@@ -647,6 +654,22 @@ class TestBuilderPassAtK:
         # 1 of 4 correct at threshold=0.7: pass^1 = C(1,1)/C(4,1) = 0.25
         assert metadata["pass_all_k"]["1"] == pytest.approx(0.25)
 
+    def test_builder_requires_example_id(self):
+        builder = GenerateOutputsBuilder(
+            env_id="test-env",
+            env_args={},
+            model="test-model",
+            client=ClientConfig(api_base_url="http://localhost:8000/v1"),
+            num_examples=1,
+            rollouts_per_example=1,
+            state_columns=[],
+            sampling_args={},
+            results_path=Path("/tmp/test-results"),
+        )
+
+        with pytest.raises(KeyError):
+            builder.add_outputs([{"reward": 1.0, "metrics": {}}])
+
 
 class TestMetricProtocol:
     def test_all_metrics_satisfy_protocol(self):

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/tests/test_v1_config_extension.py

@@ -2310,16 +2310,14 @@ def test_taskset_subclasses_inherit_registered_config_type() -> None:
 
 def test_taskset_class_loader_owns_split_loading() -> None:
     class LoaderTasksetConfig(TasksetConfig):
-        system_prompt: vf.SystemPrompt | None = "class prompt"
+        system_prompt: vf.SystemPrompt = "class prompt"
 
     class LoaderTaskset(Taskset[LoaderTasksetConfig]):
         def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
             answer = "class eval" if split == "eval" else "class tasks"
             return [{"prompt": [], "answer": answer}]
 
-        def load_system_prompt(
-            self, config: LoaderTasksetConfig
-        ) -> vf.SystemPrompt | None:
+        def load_system_prompt(self, config: LoaderTasksetConfig) -> vf.SystemPrompt:
             return config.system_prompt
 
     defaulted = LoaderTaskset(config=LoaderTasksetConfig())
@@ -2341,6 +2339,25 @@ def test_taskset_class_loader_owns_split_loading() -> None:
     assert disabled_prompt.system_prompt == []
 
 
+def test_system_prompt_alias_accepts_config_data(tmp_path) -> None:
+    prompt_path = tmp_path / "system_prompt.txt"
+    prompt_path.write_text("alias path system prompt", encoding="utf-8")
+
+    class PromptTasksetConfig(TasksetConfig):
+        system_prompt: vf.SystemPrompt = None
+
+    config = PromptTasksetConfig.model_validate(
+        {"system_prompt": {"path": str(prompt_path)}}
+    )
+    assert isinstance(config.system_prompt, vf.SystemPromptConfig)
+
+    taskset = Taskset(config=config)
+
+    assert taskset.system_prompt == [
+        {"role": "system", "content": "alias path system prompt"}
+    ]
+
+
 def test_taskset_load_tasks_can_return_empty_dataset() -> None:
     class LocalTasksetConfig(TasksetConfig):
         enabled: bool = True

verifiers-0.1.15.dev17/tests/test_v1_taskset_utils.py

@@ -0,0 +1,46 @@
+import json
+
+from datasets import Dataset
+
+from verifiers.v1.utils.taskset_utils import dataset_from_result
+
+
+def task_payload(row: dict) -> dict:
+    return json.loads(row["info"]["task"])
+
+
+def test_dataset_from_result_assigns_example_id_to_iterable_records():
+    dataset = dataset_from_result(
+        [
+            {"question": "Reverse abc.", "answer": "cba"},
+            {"question": "Reverse xyz.", "answer": "zyx"},
+        ],
+        "ReverseTextTaskset",
+    )
+
+    rows = list(dataset)
+    payloads = [task_payload(row) for row in rows]
+
+    assert [row["example_id"] for row in rows] == [0, 1]
+    assert [payload["example_id"] for payload in payloads] == [0, 1]
+    assert all(len(payload["task_id"]) == 32 for payload in payloads)
+    assert {payload["task_id"] for payload in payloads}.isdisjoint({"0", "1"})
+
+
+def test_dataset_from_result_overwrites_existing_example_id_column():
+    raw_dataset = Dataset.from_list(
+        [
+            {"question": "Reverse abc.", "answer": "cba", "example_id": None},
+            {"question": "Reverse xyz.", "answer": "zyx", "example_id": 99},
+        ]
+    )
+
+    dataset = dataset_from_result(raw_dataset, "ReverseTextTaskset")
+
+    rows = list(dataset)
+    payloads = [task_payload(row) for row in rows]
+
+    assert [row["example_id"] for row in rows] == [0, 1]
+    assert [payload["example_id"] for payload in payloads] == [0, 1]
+    assert all(len(payload["task_id"]) == 32 for payload in payloads)
+    assert {payload["task_id"] for payload in payloads}.isdisjoint({"0", "1", "99"})

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev16"
+__version__ = "0.1.15.dev17"
 
 import importlib
 import os

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/utils/metric_utils.py

@@ -162,10 +162,12 @@ class PassAtKMetric:
         self.reset()
 
     def add_output(self, output: RolloutOutput) -> None:
+        example_id = output["example_id"]
+        if example_id is None:
+            raise ValueError("output['example_id'] is required.")
         if not self._k_values:
             return
 
-        example_id = output.get("example_id", 0)
         self._example_counts[example_id] += 1
         if output.get("reward", 0.0) >= self.threshold:
             self._example_correct[example_id] += 1

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/utils/save_utils.py

@@ -218,8 +218,12 @@ def state_to_output(
     else:
         raise TypeError("state['timing'] must be a RolloutTiming or mapping.")
 
+    example_id = state["example_id"]
+    if example_id is None:
+        raise ValueError("state['example_id'] is required.")
+
     output = RolloutOutput(
-        example_id=state.get("example_id", 0),
+        example_id=example_id,
         prompt=state.get("prompt"),
         completion=state.get("completion"),
         answer=state.get("answer", ""),
@@ -671,9 +675,16 @@ class GenerateOutputsBuilder:
     def build_outputs(self, sort_by_example_id: bool = False) -> list[RolloutOutput]:
         """Return (sorted) accumulated outputs"""
         if sort_by_example_id:
-            return sorted(self.outputs, key=lambda o: o.get("example_id", 0))
+            return sorted(self.outputs, key=self.output_example_id)
         return self.outputs
 
+    @staticmethod
+    def output_example_id(output: RolloutOutput) -> int:
+        example_id = output["example_id"]
+        if example_id is None:
+            raise ValueError("output['example_id'] is required.")
+        return example_id
+
     def build(self, sort_by_example_id: bool = False) -> GenerateOutputs:
         """Build GenerateOutputs from accumulated outputs."""
         return GenerateOutputs(

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/v1/__init__.py

@@ -59,14 +59,13 @@ from .toolset import (
 )
 from .utils.endpoint_utils import Endpoint
 from .utils.binding_utils import BindingsConfig, ObjectsConfig
-from .utils.prompt_utils import SystemPromptConfig, SystemPromptStrategy
+from .utils.prompt_utils import SystemPrompt, SystemPromptConfig, SystemPromptStrategy
 from .types import (
     ConfigData,
     Handler,
     JsonData,
     Objects,
     PromptInput,
-    SystemPrompt,
     TaskSplit,
     Tasks,
 )

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/v1/harness.py

@@ -72,8 +72,8 @@ from .utils.sandbox_program_utils import (
     run_sandbox_python_program,
 )
 from .utils.prompt_utils import (
+    SystemPrompt,
     SystemPromptStrategy,
-    SystemPromptConfig,
     normalize_prompt,
     normalize_system_prompt,
     resolve_system_prompt,
@@ -88,7 +88,6 @@ from .types import (
     ConfigData,
     JsonData,
     Objects,
-    PromptInput,
 )
 
 if TYPE_CHECKING:
@@ -106,7 +105,7 @@ class HarnessConfig(LifecycleConfig):
     )
     program: ProgramConfig = ProgramConfig()
     model: ModelConfig = ModelConfig()
-    system_prompt: PromptInput | SystemPromptConfig | None = None
+    system_prompt: SystemPrompt = None
     system_prompt_strategy: SystemPromptStrategy = "HT"
     sandbox: SandboxConfig | None = None
     user: UserConfig | None = None
@@ -217,9 +216,7 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
             self.endpoint = self.load_endpoint()
             self.program = self.compile_program(self.program_config)
 
-    def load_system_prompt(
-        self, config: ConfigT
-    ) -> PromptInput | SystemPromptConfig | None:
+    def load_system_prompt(self, config: ConfigT) -> SystemPrompt:
         return config.system_prompt
 
     def load_sandbox(self, config: SandboxConfig | None) -> SandboxConfig | None:

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/v1/taskset.py

@@ -18,7 +18,7 @@ from .utils.binding_utils import (
     BindingsConfig,
     ObjectsConfig,
 )
-from .utils.prompt_utils import SystemPromptConfig, normalize_system_prompt
+from .utils.prompt_utils import SystemPrompt, normalize_system_prompt
 from .utils.config_utils import (
     coerce_config,
     config_ref_context,
@@ -36,7 +36,6 @@ from .utils.taskset_utils import (
 from .types import (
     JsonData,
     Objects,
-    PromptInput,
     TaskSplit,
     Tasks,
 )
@@ -48,7 +47,7 @@ class TasksetConfig(LifecycleConfig):
         default=None,
         validation_alias=AliasChoices("taskset_id", "id"),
     )
-    system_prompt: PromptInput | SystemPromptConfig | None = None
+    system_prompt: SystemPrompt = None
     user: UserConfig | None = None
     bindings: BindingsConfig = BindingsConfig()
     objects: ObjectsConfig = ObjectsConfig()
@@ -152,7 +151,5 @@ class Taskset(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
     def __len__(self) -> int:
         return len(self.get_dataset())
 
-    def load_system_prompt(
-        self, config: ConfigT
-    ) -> PromptInput | SystemPromptConfig | None:
+    def load_system_prompt(self, config: ConfigT) -> SystemPrompt:
         return config.system_prompt

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/v1/types.py

@@ -41,7 +41,6 @@ Tasks: TypeAlias = Dataset | Iterable[JsonData] | Iterable["Task"]
 
 PromptMessage: TypeAlias = Message | JsonData
 PromptInput: TypeAlias = str | Sequence[PromptMessage]
-SystemPrompt: TypeAlias = PromptInput
 
 ModelClient: TypeAlias = Client | ClientConfig
 RuntimeObject: TypeAlias = object

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/v1/utils/prompt_utils.py

@@ -1,7 +1,7 @@
 import importlib.util
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Literal, cast
+from typing import TYPE_CHECKING, Literal, TypeAlias, cast
 
 from pydantic import model_validator
 from typing_extensions import Self
@@ -9,7 +9,7 @@ from verifiers.types import Messages, SystemMessage
 from verifiers.utils.message_utils import normalize_messages
 
 from ..config import Config
-from ..types import JsonData, PromptInput, SystemPrompt
+from ..types import JsonData, PromptInput
 from .config_utils import current_config_ref_module
 
 if TYPE_CHECKING:
@@ -64,13 +64,15 @@ class SystemPromptConfig(Config):
     messages: list[JsonData] = []
 
     @model_validator(mode="after")
-    def validate_one_source(self) -> Self:
-        sources = [
+    def validate_one_input(self) -> Self:
+        inputs = [
             self.path is not None,
             bool(self.messages),
         ]
-        if sum(sources) != 1:
-            raise ValueError("SystemPromptConfig requires exactly one source.")
+        if sum(inputs) != 1:
+            raise ValueError(
+                "SystemPromptConfig requires exactly one of path or messages."
+            )
         return self
 
     def load(self, field_name: str) -> PromptInput | None:
@@ -81,6 +83,9 @@ class SystemPromptConfig(Config):
         return self.messages
 
 
+SystemPrompt: TypeAlias = PromptInput | SystemPromptConfig | None
+
+
 def normalize_prompt(
     value: PromptInput | None, field_name: str = "prompt"
 ) -> list[JsonData]:
@@ -95,7 +100,7 @@ def normalize_prompt(
 
 
 def normalize_system_prompt(
-    value: SystemPrompt | SystemPromptConfig | None,
+    value: SystemPrompt,
     field_name: str = "system_prompt",
 ) -> list[JsonData]:
     value = resolve_system_prompt_input(value, field_name=field_name)
@@ -111,7 +116,7 @@ def normalize_system_prompt(
 
 
 def resolve_system_prompt_input(
-    value: PromptInput | SystemPromptConfig | None,
+    value: SystemPrompt,
     *,
     field_name: str,
 ) -> PromptInput | None:

{verifiers-0.1.15.dev16 → verifiers-0.1.15.dev17}/verifiers/v1/utils/taskset_utils.py

@@ -38,10 +38,8 @@ def prepare_task(task: Task, taskset_id: str) -> Task:
         raise TypeError("v1 task loaders must return Task objects.")
     prepared = Task(cast(JsonData, dict(task)))
     prepared["taskset_id"] = taskset_id
-    if "task_id" in prepared:
+    if prepared.get("task_id") is not None:
         prepared["task_id"] = str(prepared["task_id"])
-    elif "example_id" in prepared:
-        prepared["task_id"] = str(prepared["example_id"])
     else:
         prepared["task_id"] = uuid.uuid4().hex
     return prepared.freeze()
@@ -51,13 +49,13 @@ def dataset_record_from_task(
     task: Task,
     taskset_id: str,
     index: int,
-    source: JsonData | None = None,
+    record: JsonData | None = None,
 ) -> JsonData:
     data = Task(cast(JsonData, dict(task)))
-    data.setdefault("example_id", source.get("example_id") if source else index)
+    data["example_id"] = index
     normalized = prepare_task(data, taskset_id)
     task_payload = dict(normalized)
-    dataset_record = deepcopy(dict(source or {}))
+    dataset_record = deepcopy(dict(record or {}))
     dataset_record["prompt"] = task_payload["prompt"]
     dataset_record["example_id"] = task_payload["example_id"]
     info = dataset_record.get("info")
@@ -82,9 +80,10 @@ def dataset_from_result(result: Tasks, taskset_id: str) -> Dataset:
     if isinstance(result, Dataset):
         records: list[JsonData] = []
         for index, record in enumerate(result):
-            source = cast(JsonData, dict(record))
-            task = task_from_dataset_record(source, taskset_id)
-            records.append(dataset_record_from_task(task, taskset_id, index, source))
+            row = cast(JsonData, dict(record))
+            row["example_id"] = index
+            task = task_from_dataset_record(row, taskset_id)
+            records.append(dataset_record_from_task(task, taskset_id, index, row))
         return Dataset.from_list(records)
     tasks = tasks_from_result(result, taskset_id)
     return Dataset.from_list(dataset_records_from_tasks(tasks, taskset_id))