verifiers 0.1.15.dev165__tar.gz → 0.1.15.dev167__tar.gz

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev165
+Version: 0.1.15.dev167
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_composable_env.py

@@ -187,6 +187,28 @@ def test_taskset_take():
     assert isinstance(taken, MockSandboxTaskSet)
 
 
+def test_composable_env_eval_inputs_can_shuffle_taskset_dataset():
+    taskset = MockTaskSet(dataset=_make_dataset(6), name="math")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(run_command="true"),
+    )
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=11,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=11).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]
+
+
 def test_taskset_repr():
     ts = MockTaskSet(dataset=_make_dataset(), name="mytest")
     assert "mytest" in repr(ts)

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_environment.py

@@ -673,8 +673,8 @@ class TestMaybeRetry:
         rollout_outputs = outputs["outputs"]
         assert env.call_counts[0] == 1  # No retries for non-retryable error
         assert rollout_outputs[0].get("error") is not None
-        error_info = rollout_outputs[0]["error"]
-        assert "ToolError" == error_info["error"]
+        error_data = rollout_outputs[0]["error"]
+        assert "ToolError" == error_data["error"]
 
     @pytest.mark.asyncio
     async def test_error_in_state_after_max_retries_exhausted(
@@ -694,8 +694,8 @@ class TestMaybeRetry:
         rollout_outputs = outputs["outputs"]
         assert env.call_counts[0] == 3  # 1 initial + 2 retries
         assert rollout_outputs[0].get("error") is not None
-        error_info = rollout_outputs[0]["error"]
-        assert "InfraError" == error_info["error"]
+        error_data = rollout_outputs[0]["error"]
+        assert "InfraError" == error_data["error"]
 
 
 class TestEmptyModelResponseErrors:

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_environment_extra.py

@@ -302,6 +302,30 @@ def test_evaluate_fallback_and_repeat(mock_client, make_dummy_env, make_input):
     assert len(states) == 2 * 2
 
 
+def test_get_eval_inputs_shuffles_before_take_and_repeat(mock_client, make_dummy_env):
+    ds = Dataset.from_dict(
+        {
+            "question": [f"q{i}" for i in range(6)],
+            "answer": [f"a{i}" for i in range(6)],
+        }
+    )
+    env = make_dummy_env(mock_client, dataset=ds)
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=123,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=123).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]
+
+
 @pytest.mark.asyncio
 async def test_generate_inside_running_loop(mock_client, make_dummy_env, make_input):
     env = make_dummy_env(mock_client)

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_eval_cli.py

@@ -72,6 +72,8 @@ def run_cli(make_metadata, make_state, make_input):
             "headers_from_state": None,
             "num_examples": 1,
             "rollouts_per_example": 1,
+            "shuffle": False,
+            "shuffle_seed": None,
             "max_concurrent": 1,
             "independent_scoring": False,
             "max_tokens": 42,
@@ -177,6 +179,20 @@ def test_parse_args_rejects_unknown_eval_flags():
         vf_eval.parse_args(["env1", "--unknown-flag", "value"])
 
 
+def test_cli_shuffle_defaults_seed_when_enabled(monkeypatch, run_cli):
+    captured = run_cli(
+        monkeypatch,
+        {
+            "shuffle": True,
+            "shuffle_seed": None,
+        },
+    )
+
+    config = captured["configs"][0]
+    assert config.shuffle is True
+    assert config.shuffle_seed == 0
+
+
 def test_cli_v1_env_config_overrides_preserve_env_args_config(
     tmp_path: Path, monkeypatch, run_cli
 ):
@@ -1188,6 +1204,15 @@ def test_cli_toml_per_env_rollouts_per_example(monkeypatch, run_cli):
     assert configs[1].rollouts_per_example == 5
 
 
+def test_cli_toml_per_env_shuffle(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write('[[eval]]\nenv_id = "env1"\nshuffle = true\nshuffle_seed = 321\n')
+        f.flush()
+        captured = run_cli(monkeypatch, {"env_id_or_config": f.name})
+
+    assert captured["configs"][0].shuffle_seed == 321
+
+
 def test_cli_toml_per_eval_settings_used(monkeypatch, run_cli):
     """TOML per-eval settings are used (CLI args ignored when using config)."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_path_utils.py

@@ -70,6 +70,53 @@ def test_find_latest_incomplete_eval_results_path_returns_none_when_no_match(
     assert result is None
 
 
+def test_find_latest_incomplete_eval_results_path_matches_shuffle_metadata(
+    tmp_path: Path, monkeypatch
+):
+    env_id = "dummy-env"
+    model = "openai/gpt-4.1-mini"
+    runs_dir = tmp_path / "outputs" / "evals" / f"{env_id}--{model.replace('/', '--')}"
+
+    matching_run = runs_dir / "matching"
+    wrong_seed_run = runs_dir / "wrong-seed"
+    for run in [matching_run, wrong_seed_run]:
+        run.mkdir(parents=True)
+        (run / "results.jsonl").write_text('{"example_id":0}\n', encoding="utf-8")
+
+    matching_run.joinpath("metadata.json").write_text(
+        (
+            '{"env_id":"dummy-env","model":"openai/gpt-4.1-mini",'
+            '"num_examples":4,"rollouts_per_example":1,'
+            '"shuffle":true,"shuffle_seed":123}'
+        ),
+        encoding="utf-8",
+    )
+    wrong_seed_run.joinpath("metadata.json").write_text(
+        (
+            '{"env_id":"dummy-env","model":"openai/gpt-4.1-mini",'
+            '"num_examples":4,"rollouts_per_example":1,'
+            '"shuffle":true,"shuffle_seed":456}'
+        ),
+        encoding="utf-8",
+    )
+
+    os.utime(matching_run, (1, 1))
+    os.utime(wrong_seed_run, (2, 2))
+    monkeypatch.chdir(tmp_path)
+
+    result = find_latest_incomplete_eval_results_path(
+        env_id=env_id,
+        model=model,
+        num_examples=4,
+        rollouts_per_example=1,
+        shuffle=True,
+        shuffle_seed=123,
+        env_dir_path=str(tmp_path / "environments"),
+    )
+
+    assert result.resolve() == matching_run
+
+
 def test_get_eval_runs_dir_uses_name_as_result_label(tmp_path: Path):
     runs_dir = get_eval_runs_dir(
         env_id="dummy-env",

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_save_utils.py

@@ -31,6 +31,7 @@ from verifiers.utils.save_utils import (
     load_outputs,
     make_serializable,
     save_new_outputs,
+    save_metadata,
     states_to_outputs,
     truncate_malformed_trailing_line,
     validate_resume_metadata,
@@ -199,6 +200,37 @@ class TestSavingMetadata:
             metadata["base_url"] == "http://localhost:8000/v1,http://localhost:8001/v1"
         )
 
+    def test_save_metadata_records_default_shuffle_seed_for_resume(
+        self, tmp_path: Path
+    ):
+        results_path = tmp_path / "results"
+        builder = GenerateOutputsBuilder(
+            env_id="math-env",
+            env_args={},
+            model="test-model",
+            client=ClientConfig(api_base_url="http://localhost:8000/v1"),
+            num_examples=3,
+            rollouts_per_example=2,
+            state_columns=[],
+            sampling_args={},
+            results_path=results_path,
+            shuffle=True,
+        )
+        save_metadata(builder.build_metadata(), results_path)
+
+        assert (
+            json.loads((results_path / "metadata.json").read_text())["shuffle_seed"]
+            == 0
+        )
+        validate_resume_metadata(
+            results_path=results_path,
+            env_id="math-env",
+            model="test-model",
+            num_examples=3,
+            rollouts_per_example=2,
+            shuffle=True,
+        )
+
 
 class TestSavingResults:
     def test_response_usage_tokens_prompt_completion(self):
@@ -595,6 +627,35 @@ class TestResumeMetadataValidation:
                 rollouts_per_example=2,
             )
 
+    def test_validate_resume_metadata_rejects_shuffle_mismatch(self, tmp_path: Path):
+        results_path = tmp_path / "results"
+        results_path.mkdir()
+        metadata_path = results_path / "metadata.json"
+        metadata_path.write_text(
+            json.dumps(
+                {
+                    "env_id": "math-env",
+                    "model": "test-model",
+                    "num_examples": 3,
+                    "rollouts_per_example": 2,
+                    "shuffle": True,
+                    "shuffle_seed": 123,
+                }
+            ),
+            encoding="utf-8",
+        )
+
+        with pytest.raises(ValueError, match="shuffle_seed"):
+            validate_resume_metadata(
+                results_path=results_path,
+                env_id="math-env",
+                model="test-model",
+                num_examples=3,
+                rollouts_per_example=2,
+                shuffle=True,
+                shuffle_seed=456,
+            )
+
 
 class TestBuilderPassAtK:
     """Tests for pass@k integration in GenerateOutputsBuilder."""

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_v1_config_extension.py

@@ -1156,12 +1156,13 @@ def test_lifecycle_fields_are_framework_managed() -> None:
     task = Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
     state = vf.State.for_task(task)
     assert state.uses_v1_contract is True
+    error = vf.Error("boom")
 
     for key, value in {
         "is_completed": True,
         "stop_condition": "done",
         "is_truncated": True,
-        "error": {"message": "boom"},
+        "error": error,
     }.items():
         assert State({key: value})[key] == value
         with pytest.raises(RuntimeError, match="framework-managed"):
@@ -1188,12 +1189,14 @@ def test_lifecycle_fields_are_framework_managed() -> None:
     state._set_completed(True)
     state._set_stop_condition("done")
     state._set_truncated(True)
-    state._set_error({"message": "boom"})
+    with pytest.raises(TypeError, match="vf.Error"):
+        state._set_error(cast(Any, {"message": "boom"}))
+    state._set_error(error)
 
     assert state["is_completed"] is True
     assert state["stop_condition"] == "done"
     assert state["is_truncated"] is True
-    assert state["error"] == {"message": "boom"}
+    assert state["error"] is error
 
 
 def test_toolsets_config_accepts_addressable_map_and_fn_tables() -> None:

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_v1_runtime_lifecycle.py

@@ -1646,14 +1646,19 @@ def test_sandbox_program_patch_cannot_set_lifecycle_fields() -> None:
     patch = {
         "stop_condition": "no_tools",
         "is_truncated": True,
-        "error": {"message": "handled"},
+        "error": vf.ErrorData(
+            error="SandboxError",
+            message="handled",
+            error_chain_repr="SandboxError('handled')",
+            error_chain_str="SandboxError",
+        ),
     }
     apply_internal_state_patch(state, patch, mode="base")
 
     assert patch == {}
     assert state["stop_condition"] == "no_tools"
     assert state["is_truncated"] is True
-    assert state["error"] == {"message": "handled"}
+    assert isinstance(state["error"], vf.SandboxError)
 
 
 def test_program_channels_mcp_injects_proxy_into_sandbox_program() -> None:

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/tests/test_v1_taskset_utils.py

@@ -4,7 +4,7 @@ import types
 
 from datasets import Dataset
 
-from verifiers.v1 import Taskset
+from verifiers.v1 import Env, Taskset
 from verifiers.v1.utils.taskset_utils import dataset_from_result, discover_sibling_dir
 
 
@@ -118,3 +118,25 @@ def test_taskset_skips_empty_skills_dir(tmp_path, monkeypatch) -> None:
 
     assert taskset.get_skills_dir() == skills_dir
     assert taskset.get_upload_dirs() == {"skills": skills_dir}
+
+
+def test_v1_env_eval_inputs_can_shuffle_taskset_dataset() -> None:
+    class DemoTaskset(Taskset):
+        def load_tasks(self, split: str = "train"):
+            return [{"question": f"Reverse {i}.", "answer": str(i)} for i in range(6)]
+
+    env = Env(taskset=DemoTaskset())
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=7,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=7).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/verifiers/envs/environment.py

@@ -400,12 +400,13 @@ class Environment(ABC):
         self.build_dataset()
         if self.dataset is None:
             raise ValueError("dataset is not set")
+        dataset = self.dataset
         if seed is not None:
-            self.dataset = self.dataset.shuffle(seed=seed)
+            dataset = dataset.shuffle(seed=seed)
         if n > 0:
-            n = min(n, len(self.dataset))
-            return self.dataset.select(range(n))
-        return self.dataset
+            n = min(n, len(dataset))
+            return dataset.select(range(n))
+        return dataset
 
     @final
     def get_eval_dataset(self, n: int = -1, seed: int | None = None) -> "Dataset":
@@ -415,12 +416,13 @@ class Environment(ABC):
                 "eval_dataset is not set, falling back to train dataset"
             )
             return self.get_dataset(n, seed)
+        eval_dataset = self.eval_dataset
         if seed is not None:
-            self.eval_dataset = self.eval_dataset.shuffle(seed=seed)
+            eval_dataset = eval_dataset.shuffle(seed=seed)
         if n > 0:
-            n = min(n, len(self.eval_dataset))
-            return self.eval_dataset.select(range(n))
-        return self.eval_dataset
+            n = min(n, len(eval_dataset))
+            return eval_dataset.select(range(n))
+        return eval_dataset
 
     @final
     def _get_usage_tracker(
@@ -831,6 +833,8 @@ class Environment(ABC):
         hf_hub_dataset_name: str | None = None,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
         on_start: StartCallback | None = None,
         on_progress: ProgressCallback | list[ProgressCallback] | None = None,
         on_log: LogCallback | None = None,
@@ -934,6 +938,9 @@ class Environment(ABC):
         total_rollouts = len(raw_inputs)
         num_examples = len(set([i["example_id"] for i in raw_inputs]))
         rollouts_per_example = total_rollouts // num_examples if num_examples > 0 else 0
+        resolved_shuffle_seed = (
+            (0 if shuffle_seed is None else shuffle_seed) if shuffle else None
+        )
         builder = GenerateOutputsBuilder(
             env_id=self.env_id,
             env_args=self.env_args,
@@ -945,6 +952,8 @@ class Environment(ABC):
             sampling_args=sampling_args,
             results_path=results_path,
             pass_threshold=self.pass_threshold,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
         )
 
         single_client: Client | None = None
@@ -989,6 +998,8 @@ class Environment(ABC):
                     model=model,
                     num_examples=num_examples,
                     rollouts_per_example=rollouts_per_example,
+                    shuffle=shuffle,
+                    shuffle_seed=resolved_shuffle_seed,
                 )
                 on_log(f"Resuming evaluation from {results_path}")
                 outputs = load_outputs(results_path)
@@ -1152,11 +1163,20 @@ class Environment(ABC):
 
     # evaluation
     def _get_eval_inputs(
-        self, num_examples: int = -1, rollouts_per_example: int = 1
+        self,
+        num_examples: int = -1,
+        rollouts_per_example: int = 1,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
     ) -> List[RolloutInput]:
         # get_eval_dataset handles fallback to train dataset if no eval source exists
-        inputs = self.get_eval_dataset(n=num_examples)
+        inputs = self.get_eval_dataset()
         assert inputs is not None, "No dataset found"
+        if shuffle:
+            inputs = inputs.shuffle(seed=0 if shuffle_seed is None else shuffle_seed)
+        if num_examples > 0:
+            num_examples = min(num_examples, len(inputs))
+            inputs = inputs.select(range(num_examples))
         if rollouts_per_example > 1:
             inputs = inputs.repeat(rollouts_per_example)
         return inputs.to_list()
@@ -1176,6 +1196,8 @@ class Environment(ABC):
         hf_hub_dataset_name: str | None = None,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
         on_start: StartCallback | None = None,
         on_progress: ProgressCallback | list[ProgressCallback] | None = None,
         on_log: LogCallback | None = None,
@@ -1189,7 +1211,13 @@ class Environment(ABC):
                 A single callback replaces the default. A list of callbacks runs
                 alongside the default.
         """
-        inputs = self._get_eval_inputs(num_examples, rollouts_per_example)
+        resolved_shuffle_seed = 0 if shuffle and shuffle_seed is None else shuffle_seed
+        inputs = self._get_eval_inputs(
+            num_examples,
+            rollouts_per_example,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
+        )
         return await self.generate(
             inputs,
             client=client,
@@ -1203,6 +1231,8 @@ class Environment(ABC):
             hf_hub_dataset_name=hf_hub_dataset_name,
             independent_scoring=independent_scoring,
             max_retries=max_retries,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
             on_start=on_start,
             on_progress=on_progress,
             on_log=on_log,
@@ -1224,11 +1254,19 @@ class Environment(ABC):
         hf_hub_dataset_name: str | None = None,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
     ) -> GenerateOutputs:
         """
         Evaluate model on the Environment evaluation dataset synchronously.
         """
-        inputs = self._get_eval_inputs(num_examples, rollouts_per_example)
+        resolved_shuffle_seed = 0 if shuffle and shuffle_seed is None else shuffle_seed
+        inputs = self._get_eval_inputs(
+            num_examples,
+            rollouts_per_example,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
+        )
         return self.generate_sync(
             inputs,
             client=client,
@@ -1242,6 +1280,8 @@ class Environment(ABC):
             hf_hub_dataset_name=hf_hub_dataset_name,
             independent_scoring=independent_scoring,
             max_retries=max_retries,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
         )
 
     # setters for use by trainers

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/verifiers/envs/experimental/cli_agent_env.py

@@ -717,7 +717,7 @@ class CliAgentEnv(SandboxMixin, vf.MultiTurnEnv):
         num_turns = len(state.get("trajectory", []))
         stop_condition = state.get("stop_condition", "unknown")
         error = state.get("error")
-        error_info = (
+        error_summary = (
             f"{type(error).__name__}: {truncate(str(error), 80)}" if error else None
         )
         exit_code = state.get("agent_exit_code")
@@ -741,8 +741,8 @@ class CliAgentEnv(SandboxMixin, vf.MultiTurnEnv):
         ]
         if timed_out:
             parts.append("timed_out=True")
-        if error_info:
-            parts.append(f"error={error_info}")
+        if error_summary:
+            parts.append(f"error={error_summary}")
         self.logger.info(" | ".join(parts))
 
     @vf.cleanup

{verifiers-0.1.15.dev165 → verifiers-0.1.15.dev167}/verifiers/scripts/eval.py

@@ -473,6 +473,18 @@ def build_parser() -> argparse.ArgumentParser:
         default=None,
         help="Number of rollouts per example",
     )
+    parser.add_argument(
+        "--shuffle",
+        default=False,
+        action="store_true",
+        help="Shuffle the evaluation dataset before selecting examples",
+    )
+    parser.add_argument(
+        "--shuffle-seed",
+        type=int,
+        default=None,
+        help="Seed for --shuffle. Defaults to 0 when --shuffle is enabled.",
+    )
     parser.add_argument(
         "--max-concurrent",
         "-c",
@@ -712,6 +724,12 @@ def main(argv: list[str] | None = None):
             logger.debug(
                 f"Using rollouts_per_example={rollouts_per_example} from {source}"
             )
+        shuffle = bool(raw.get("shuffle", False))
+        shuffle_seed = raw.get("shuffle_seed")
+        if shuffle and shuffle_seed is None:
+            shuffle_seed = 0
+        if not shuffle:
+            shuffle_seed = None
 
         raw_endpoint_id = raw.get("endpoint_id")
         raw_model_field = raw.get("model")
@@ -918,6 +936,8 @@ def main(argv: list[str] | None = None):
                 model=model,
                 num_examples=num_examples,
                 rollouts_per_example=rollouts_per_example,
+                shuffle=shuffle,
+                shuffle_seed=shuffle_seed,
                 env_dir_path=raw.get("env_dir_path", DEFAULT_ENV_DIR_PATH),
                 output_dir=raw.get("output_dir"),
                 name=name,
@@ -957,6 +977,8 @@ def main(argv: list[str] | None = None):
             sampling_args=merged_sampling_args,
             num_examples=num_examples,
             rollouts_per_example=rollouts_per_example,
+            shuffle=shuffle,
+            shuffle_seed=shuffle_seed,
             max_concurrent=raw.get("max_concurrent", DEFAULT_MAX_CONCURRENT),
             max_retries=raw.get("max_retries", 3),
             num_workers=raw.get("num_workers", "auto"),