verifiers 0.1.15.dev166__tar.gz → 0.1.15.dev167__tar.gz

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev166
+Version: 0.1.15.dev167
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/tests/test_composable_env.py

@@ -187,6 +187,28 @@ def test_taskset_take():
     assert isinstance(taken, MockSandboxTaskSet)
 
 
+def test_composable_env_eval_inputs_can_shuffle_taskset_dataset():
+    taskset = MockTaskSet(dataset=_make_dataset(6), name="math")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(run_command="true"),
+    )
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=11,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=11).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]
+
+
 def test_taskset_repr():
     ts = MockTaskSet(dataset=_make_dataset(), name="mytest")
     assert "mytest" in repr(ts)

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/tests/test_environment_extra.py

@@ -302,6 +302,30 @@ def test_evaluate_fallback_and_repeat(mock_client, make_dummy_env, make_input):
     assert len(states) == 2 * 2
 
 
+def test_get_eval_inputs_shuffles_before_take_and_repeat(mock_client, make_dummy_env):
+    ds = Dataset.from_dict(
+        {
+            "question": [f"q{i}" for i in range(6)],
+            "answer": [f"a{i}" for i in range(6)],
+        }
+    )
+    env = make_dummy_env(mock_client, dataset=ds)
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=123,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=123).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]
+
+
 @pytest.mark.asyncio
 async def test_generate_inside_running_loop(mock_client, make_dummy_env, make_input):
     env = make_dummy_env(mock_client)

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/tests/test_eval_cli.py

@@ -72,6 +72,8 @@ def run_cli(make_metadata, make_state, make_input):
             "headers_from_state": None,
             "num_examples": 1,
             "rollouts_per_example": 1,
+            "shuffle": False,
+            "shuffle_seed": None,
             "max_concurrent": 1,
             "independent_scoring": False,
             "max_tokens": 42,
@@ -177,6 +179,20 @@ def test_parse_args_rejects_unknown_eval_flags():
         vf_eval.parse_args(["env1", "--unknown-flag", "value"])
 
 
+def test_cli_shuffle_defaults_seed_when_enabled(monkeypatch, run_cli):
+    captured = run_cli(
+        monkeypatch,
+        {
+            "shuffle": True,
+            "shuffle_seed": None,
+        },
+    )
+
+    config = captured["configs"][0]
+    assert config.shuffle is True
+    assert config.shuffle_seed == 0
+
+
 def test_cli_v1_env_config_overrides_preserve_env_args_config(
     tmp_path: Path, monkeypatch, run_cli
 ):
@@ -1188,6 +1204,15 @@ def test_cli_toml_per_env_rollouts_per_example(monkeypatch, run_cli):
     assert configs[1].rollouts_per_example == 5
 
 
+def test_cli_toml_per_env_shuffle(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write('[[eval]]\nenv_id = "env1"\nshuffle = true\nshuffle_seed = 321\n')
+        f.flush()
+        captured = run_cli(monkeypatch, {"env_id_or_config": f.name})
+
+    assert captured["configs"][0].shuffle_seed == 321
+
+
 def test_cli_toml_per_eval_settings_used(monkeypatch, run_cli):
     """TOML per-eval settings are used (CLI args ignored when using config)."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/tests/test_path_utils.py

@@ -70,6 +70,53 @@ def test_find_latest_incomplete_eval_results_path_returns_none_when_no_match(
     assert result is None
 
 
+def test_find_latest_incomplete_eval_results_path_matches_shuffle_metadata(
+    tmp_path: Path, monkeypatch
+):
+    env_id = "dummy-env"
+    model = "openai/gpt-4.1-mini"
+    runs_dir = tmp_path / "outputs" / "evals" / f"{env_id}--{model.replace('/', '--')}"
+
+    matching_run = runs_dir / "matching"
+    wrong_seed_run = runs_dir / "wrong-seed"
+    for run in [matching_run, wrong_seed_run]:
+        run.mkdir(parents=True)
+        (run / "results.jsonl").write_text('{"example_id":0}\n', encoding="utf-8")
+
+    matching_run.joinpath("metadata.json").write_text(
+        (
+            '{"env_id":"dummy-env","model":"openai/gpt-4.1-mini",'
+            '"num_examples":4,"rollouts_per_example":1,'
+            '"shuffle":true,"shuffle_seed":123}'
+        ),
+        encoding="utf-8",
+    )
+    wrong_seed_run.joinpath("metadata.json").write_text(
+        (
+            '{"env_id":"dummy-env","model":"openai/gpt-4.1-mini",'
+            '"num_examples":4,"rollouts_per_example":1,'
+            '"shuffle":true,"shuffle_seed":456}'
+        ),
+        encoding="utf-8",
+    )
+
+    os.utime(matching_run, (1, 1))
+    os.utime(wrong_seed_run, (2, 2))
+    monkeypatch.chdir(tmp_path)
+
+    result = find_latest_incomplete_eval_results_path(
+        env_id=env_id,
+        model=model,
+        num_examples=4,
+        rollouts_per_example=1,
+        shuffle=True,
+        shuffle_seed=123,
+        env_dir_path=str(tmp_path / "environments"),
+    )
+
+    assert result.resolve() == matching_run
+
+
 def test_get_eval_runs_dir_uses_name_as_result_label(tmp_path: Path):
     runs_dir = get_eval_runs_dir(
         env_id="dummy-env",

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/tests/test_save_utils.py

@@ -31,6 +31,7 @@ from verifiers.utils.save_utils import (
     load_outputs,
     make_serializable,
     save_new_outputs,
+    save_metadata,
     states_to_outputs,
     truncate_malformed_trailing_line,
     validate_resume_metadata,
@@ -199,6 +200,37 @@ class TestSavingMetadata:
             metadata["base_url"] == "http://localhost:8000/v1,http://localhost:8001/v1"
         )
 
+    def test_save_metadata_records_default_shuffle_seed_for_resume(
+        self, tmp_path: Path
+    ):
+        results_path = tmp_path / "results"
+        builder = GenerateOutputsBuilder(
+            env_id="math-env",
+            env_args={},
+            model="test-model",
+            client=ClientConfig(api_base_url="http://localhost:8000/v1"),
+            num_examples=3,
+            rollouts_per_example=2,
+            state_columns=[],
+            sampling_args={},
+            results_path=results_path,
+            shuffle=True,
+        )
+        save_metadata(builder.build_metadata(), results_path)
+
+        assert (
+            json.loads((results_path / "metadata.json").read_text())["shuffle_seed"]
+            == 0
+        )
+        validate_resume_metadata(
+            results_path=results_path,
+            env_id="math-env",
+            model="test-model",
+            num_examples=3,
+            rollouts_per_example=2,
+            shuffle=True,
+        )
+
 
 class TestSavingResults:
     def test_response_usage_tokens_prompt_completion(self):
@@ -595,6 +627,35 @@ class TestResumeMetadataValidation:
                 rollouts_per_example=2,
             )
 
+    def test_validate_resume_metadata_rejects_shuffle_mismatch(self, tmp_path: Path):
+        results_path = tmp_path / "results"
+        results_path.mkdir()
+        metadata_path = results_path / "metadata.json"
+        metadata_path.write_text(
+            json.dumps(
+                {
+                    "env_id": "math-env",
+                    "model": "test-model",
+                    "num_examples": 3,
+                    "rollouts_per_example": 2,
+                    "shuffle": True,
+                    "shuffle_seed": 123,
+                }
+            ),
+            encoding="utf-8",
+        )
+
+        with pytest.raises(ValueError, match="shuffle_seed"):
+            validate_resume_metadata(
+                results_path=results_path,
+                env_id="math-env",
+                model="test-model",
+                num_examples=3,
+                rollouts_per_example=2,
+                shuffle=True,
+                shuffle_seed=456,
+            )
+
 
 class TestBuilderPassAtK:
     """Tests for pass@k integration in GenerateOutputsBuilder."""

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/tests/test_v1_taskset_utils.py

@@ -4,7 +4,7 @@ import types
 
 from datasets import Dataset
 
-from verifiers.v1 import Taskset
+from verifiers.v1 import Env, Taskset
 from verifiers.v1.utils.taskset_utils import dataset_from_result, discover_sibling_dir
 
 
@@ -118,3 +118,25 @@ def test_taskset_skips_empty_skills_dir(tmp_path, monkeypatch) -> None:
 
     assert taskset.get_skills_dir() == skills_dir
     assert taskset.get_upload_dirs() == {"skills": skills_dir}
+
+
+def test_v1_env_eval_inputs_can_shuffle_taskset_dataset() -> None:
+    class DemoTaskset(Taskset):
+        def load_tasks(self, split: str = "train"):
+            return [{"question": f"Reverse {i}.", "answer": str(i)} for i in range(6)]
+
+    env = Env(taskset=DemoTaskset())
+
+    inputs = env._get_eval_inputs(
+        num_examples=3,
+        rollouts_per_example=2,
+        shuffle=True,
+        shuffle_seed=7,
+    )
+    expected = (
+        env.get_eval_dataset().shuffle(seed=7).select(range(3)).repeat(2).to_list()
+    )
+
+    assert [row["example_id"] for row in inputs] == [
+        row["example_id"] for row in expected
+    ]

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/verifiers/envs/environment.py

@@ -400,12 +400,13 @@ class Environment(ABC):
         self.build_dataset()
         if self.dataset is None:
             raise ValueError("dataset is not set")
+        dataset = self.dataset
         if seed is not None:
-            self.dataset = self.dataset.shuffle(seed=seed)
+            dataset = dataset.shuffle(seed=seed)
         if n > 0:
-            n = min(n, len(self.dataset))
-            return self.dataset.select(range(n))
-        return self.dataset
+            n = min(n, len(dataset))
+            return dataset.select(range(n))
+        return dataset
 
     @final
     def get_eval_dataset(self, n: int = -1, seed: int | None = None) -> "Dataset":
@@ -415,12 +416,13 @@ class Environment(ABC):
                 "eval_dataset is not set, falling back to train dataset"
             )
             return self.get_dataset(n, seed)
+        eval_dataset = self.eval_dataset
         if seed is not None:
-            self.eval_dataset = self.eval_dataset.shuffle(seed=seed)
+            eval_dataset = eval_dataset.shuffle(seed=seed)
         if n > 0:
-            n = min(n, len(self.eval_dataset))
-            return self.eval_dataset.select(range(n))
-        return self.eval_dataset
+            n = min(n, len(eval_dataset))
+            return eval_dataset.select(range(n))
+        return eval_dataset
 
     @final
     def _get_usage_tracker(
@@ -831,6 +833,8 @@ class Environment(ABC):
         hf_hub_dataset_name: str | None = None,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
         on_start: StartCallback | None = None,
         on_progress: ProgressCallback | list[ProgressCallback] | None = None,
         on_log: LogCallback | None = None,
@@ -934,6 +938,9 @@ class Environment(ABC):
         total_rollouts = len(raw_inputs)
         num_examples = len(set([i["example_id"] for i in raw_inputs]))
         rollouts_per_example = total_rollouts // num_examples if num_examples > 0 else 0
+        resolved_shuffle_seed = (
+            (0 if shuffle_seed is None else shuffle_seed) if shuffle else None
+        )
         builder = GenerateOutputsBuilder(
             env_id=self.env_id,
             env_args=self.env_args,
@@ -945,6 +952,8 @@ class Environment(ABC):
             sampling_args=sampling_args,
             results_path=results_path,
             pass_threshold=self.pass_threshold,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
         )
 
         single_client: Client | None = None
@@ -989,6 +998,8 @@ class Environment(ABC):
                     model=model,
                     num_examples=num_examples,
                     rollouts_per_example=rollouts_per_example,
+                    shuffle=shuffle,
+                    shuffle_seed=resolved_shuffle_seed,
                 )
                 on_log(f"Resuming evaluation from {results_path}")
                 outputs = load_outputs(results_path)
@@ -1152,11 +1163,20 @@ class Environment(ABC):
 
     # evaluation
     def _get_eval_inputs(
-        self, num_examples: int = -1, rollouts_per_example: int = 1
+        self,
+        num_examples: int = -1,
+        rollouts_per_example: int = 1,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
     ) -> List[RolloutInput]:
         # get_eval_dataset handles fallback to train dataset if no eval source exists
-        inputs = self.get_eval_dataset(n=num_examples)
+        inputs = self.get_eval_dataset()
         assert inputs is not None, "No dataset found"
+        if shuffle:
+            inputs = inputs.shuffle(seed=0 if shuffle_seed is None else shuffle_seed)
+        if num_examples > 0:
+            num_examples = min(num_examples, len(inputs))
+            inputs = inputs.select(range(num_examples))
         if rollouts_per_example > 1:
             inputs = inputs.repeat(rollouts_per_example)
         return inputs.to_list()
@@ -1176,6 +1196,8 @@ class Environment(ABC):
         hf_hub_dataset_name: str | None = None,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
         on_start: StartCallback | None = None,
         on_progress: ProgressCallback | list[ProgressCallback] | None = None,
         on_log: LogCallback | None = None,
@@ -1189,7 +1211,13 @@ class Environment(ABC):
                 A single callback replaces the default. A list of callbacks runs
                 alongside the default.
         """
-        inputs = self._get_eval_inputs(num_examples, rollouts_per_example)
+        resolved_shuffle_seed = 0 if shuffle and shuffle_seed is None else shuffle_seed
+        inputs = self._get_eval_inputs(
+            num_examples,
+            rollouts_per_example,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
+        )
         return await self.generate(
             inputs,
             client=client,
@@ -1203,6 +1231,8 @@ class Environment(ABC):
             hf_hub_dataset_name=hf_hub_dataset_name,
             independent_scoring=independent_scoring,
             max_retries=max_retries,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
             on_start=on_start,
             on_progress=on_progress,
             on_log=on_log,
@@ -1224,11 +1254,19 @@ class Environment(ABC):
         hf_hub_dataset_name: str | None = None,
         independent_scoring: bool = False,
         max_retries: int = 0,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
     ) -> GenerateOutputs:
         """
         Evaluate model on the Environment evaluation dataset synchronously.
         """
-        inputs = self._get_eval_inputs(num_examples, rollouts_per_example)
+        resolved_shuffle_seed = 0 if shuffle and shuffle_seed is None else shuffle_seed
+        inputs = self._get_eval_inputs(
+            num_examples,
+            rollouts_per_example,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
+        )
         return self.generate_sync(
             inputs,
             client=client,
@@ -1242,6 +1280,8 @@ class Environment(ABC):
             hf_hub_dataset_name=hf_hub_dataset_name,
             independent_scoring=independent_scoring,
             max_retries=max_retries,
+            shuffle=shuffle,
+            shuffle_seed=resolved_shuffle_seed,
         )
 
     # setters for use by trainers

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/verifiers/scripts/eval.py

@@ -473,6 +473,18 @@ def build_parser() -> argparse.ArgumentParser:
         default=None,
         help="Number of rollouts per example",
     )
+    parser.add_argument(
+        "--shuffle",
+        default=False,
+        action="store_true",
+        help="Shuffle the evaluation dataset before selecting examples",
+    )
+    parser.add_argument(
+        "--shuffle-seed",
+        type=int,
+        default=None,
+        help="Seed for --shuffle. Defaults to 0 when --shuffle is enabled.",
+    )
     parser.add_argument(
         "--max-concurrent",
         "-c",
@@ -712,6 +724,12 @@ def main(argv: list[str] | None = None):
             logger.debug(
                 f"Using rollouts_per_example={rollouts_per_example} from {source}"
             )
+        shuffle = bool(raw.get("shuffle", False))
+        shuffle_seed = raw.get("shuffle_seed")
+        if shuffle and shuffle_seed is None:
+            shuffle_seed = 0
+        if not shuffle:
+            shuffle_seed = None
 
         raw_endpoint_id = raw.get("endpoint_id")
         raw_model_field = raw.get("model")
@@ -918,6 +936,8 @@ def main(argv: list[str] | None = None):
                 model=model,
                 num_examples=num_examples,
                 rollouts_per_example=rollouts_per_example,
+                shuffle=shuffle,
+                shuffle_seed=shuffle_seed,
                 env_dir_path=raw.get("env_dir_path", DEFAULT_ENV_DIR_PATH),
                 output_dir=raw.get("output_dir"),
                 name=name,
@@ -957,6 +977,8 @@ def main(argv: list[str] | None = None):
             sampling_args=merged_sampling_args,
             num_examples=num_examples,
             rollouts_per_example=rollouts_per_example,
+            shuffle=shuffle,
+            shuffle_seed=shuffle_seed,
             max_concurrent=raw.get("max_concurrent", DEFAULT_MAX_CONCURRENT),
             max_retries=raw.get("max_retries", 3),
             num_workers=raw.get("num_workers", "auto"),

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/verifiers/types.py

@@ -1152,6 +1152,8 @@ class GenerateMetadata(TypedDict):
     base_url: str
     num_examples: int
     rollouts_per_example: int
+    shuffle: NotRequired[bool]
+    shuffle_seed: NotRequired[int | None]
     sampling_args: SamplingArgs
     date: str
     time: float  # whole-eval wall-clock seconds
@@ -1340,6 +1342,8 @@ class EvalConfig(BaseModel):
     sampling_args: SamplingArgs
     num_examples: int
     rollouts_per_example: int
+    shuffle: bool = False
+    shuffle_seed: int | None = None
     max_concurrent: int
     num_workers: int | str = "auto"
     independent_scoring: bool = False

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/verifiers/utils/eval_utils.py

@@ -658,6 +658,8 @@ def load_toml_config(
         # evaluation
         "num_examples",
         "rollouts_per_example",
+        "shuffle",
+        "shuffle_seed",
         "max_concurrent",
         "independent_scoring",
         "max_retries",
@@ -1102,7 +1104,7 @@ async def run_evaluation(
 
         logger.debug(f"Starting evaluation with model: {config.model}")
         logger.debug(
-            f"Configuration: num_examples={config.num_examples}, rollouts_per_example={config.rollouts_per_example}, max_concurrent={config.max_concurrent}"
+            f"Configuration: num_examples={config.num_examples}, rollouts_per_example={config.rollouts_per_example}, shuffle={config.shuffle}, shuffle_seed={config.shuffle_seed}, max_concurrent={config.max_concurrent}"
         )
 
         effective_group_max_concurrent = config.max_concurrent
@@ -1127,6 +1129,8 @@ async def run_evaluation(
             sampling_args=config.sampling_args,
             num_examples=config.num_examples,
             rollouts_per_example=config.rollouts_per_example,
+            shuffle=config.shuffle,
+            shuffle_seed=config.shuffle_seed,
             max_concurrent=effective_group_max_concurrent,
             results_path=results_path,
             state_columns=config.state_columns,

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/verifiers/utils/path_utils.py

@@ -107,6 +107,8 @@ def find_latest_incomplete_eval_results_path(
     model: str,
     num_examples: int,
     rollouts_per_example: int,
+    shuffle: bool = False,
+    shuffle_seed: int | None = None,
     env_dir_path: str = "./environments",
     output_dir: str | None = None,
     name: str | None = None,
@@ -147,6 +149,12 @@ def find_latest_incomplete_eval_results_path(
             continue
         if metadata.get("rollouts_per_example") != rollouts_per_example:
             continue
+        if bool(metadata.get("shuffle", False)) != shuffle:
+            continue
+        if shuffle:
+            expected_shuffle_seed = 0 if shuffle_seed is None else shuffle_seed
+            if metadata.get("shuffle_seed") != expected_shuffle_seed:
+                continue
 
         saved_num_examples = metadata.get("num_examples")
         if not isinstance(saved_num_examples, int) or saved_num_examples > num_examples:

{verifiers-0.1.15.dev166 → verifiers-0.1.15.dev167}/verifiers/utils/save_utils.py

@@ -550,6 +550,8 @@ class GenerateOutputsBuilder:
         sampling_args: SamplingArgs,
         results_path: Path | None,
         pass_threshold: float = 0.5,
+        shuffle: bool = False,
+        shuffle_seed: int | None = None,
     ):
         self.env_id = env_id
         self.env_args = env_args
@@ -557,6 +559,10 @@ class GenerateOutputsBuilder:
         self.client = client
         self.num_examples = num_examples
         self.rollouts_per_example = rollouts_per_example
+        self.shuffle = shuffle
+        self.shuffle_seed = (
+            (0 if shuffle_seed is None else shuffle_seed) if shuffle else None
+        )
         self.state_columns = state_columns or []
         self.sampling_args = sampling_args
         self.results_path = results_path or get_results_path(env_id, model)
@@ -659,6 +665,8 @@ class GenerateOutputsBuilder:
             base_url=self.base_url,
             num_examples=self.num_examples,
             rollouts_per_example=self.rollouts_per_example,
+            shuffle=self.shuffle,
+            shuffle_seed=self.shuffle_seed,
             sampling_args=self.sampling_args,
             date=datetime.now().isoformat(),
             time=time.time() - self.start_time,
@@ -733,6 +741,8 @@ def validate_resume_metadata(
     model: str,
     num_examples: int,
     rollouts_per_example: int,
+    shuffle: bool = False,
+    shuffle_seed: int | None = None,
 ) -> None:
     """Validate saved metadata matches the current resume configuration.
 
@@ -778,6 +788,17 @@ def validate_resume_metadata(
             f"num_examples: saved={saved_num_examples!r}, current={num_examples!r} (current must be >= saved)"
         )
 
+    saved_shuffle = bool(saved_metadata.get("shuffle", False))
+    if saved_shuffle != shuffle:
+        mismatches.append(f"shuffle: saved={saved_shuffle!r}, current={shuffle!r}")
+    if shuffle:
+        expected_shuffle_seed = 0 if shuffle_seed is None else shuffle_seed
+        saved_shuffle_seed = saved_metadata.get("shuffle_seed", "<missing>")
+        if saved_shuffle_seed != expected_shuffle_seed:
+            mismatches.append(
+                f"shuffle_seed: saved={saved_shuffle_seed!r}, current={expected_shuffle_seed!r}"
+            )
+
     if mismatches:
         mismatch_text = "; ".join(mismatches)
         raise ValueError(