verifiers 0.1.15.dev171__tar.gz → 0.1.15.dev173__tar.gz

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev171
+Version: 0.1.15.dev173
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/tests/test_environment_extra.py

@@ -557,7 +557,12 @@ async def test_generate_resume_closes_local_endpoint_clients(
     results_path = tmp_path / "resume-complete"
     results_path.mkdir()
     (results_path / "results.jsonl").write_text(
-        json.dumps(make_output(example_id=0)) + "\n",
+        (
+            json.dumps(make_output(example_id=0, reward=0.0))
+            + "\n"
+            + json.dumps(make_output(example_id=0, reward=1.0))
+            + "\n"
+        ),
         encoding="utf-8",
     )
     (results_path / "metadata.json").write_text(
@@ -583,9 +588,12 @@ async def test_generate_resume_closes_local_endpoint_clients(
         ),
         model="test-model",
         results_path=results_path,
+        save_results=True,
     )
 
     assert len(outputs["outputs"]) == 1
+    saved_metadata = json.loads((results_path / "metadata.json").read_text())
+    assert saved_metadata["avg_reward"] == 0.0
     assert len(created_clients) == 2
     assert all(client.closed for client in created_clients)
 
@@ -669,6 +677,21 @@ async def test_generate_resume_raises_on_metadata_mismatch(
 ):
     env = make_dummy_env(mock_client)
 
+    invalid_results_path = tmp_path / "missing-metadata"
+    invalid_results_path.mkdir()
+    (invalid_results_path / "results.jsonl").write_text(
+        json.dumps({"example_id": 99, "label": "existing"}) + "\n",
+        encoding="utf-8",
+    )
+    with pytest.raises(ValueError, match="already exists without valid metadata"):
+        await env.generate(
+            inputs=[make_input(example_id=0)],
+            client=mock_client,
+            model="test-model",
+            results_path=invalid_results_path,
+            save_results=True,
+        )
+
     results_path = tmp_path / "resume"
     results_path.mkdir()
     (results_path / "results.jsonl").write_text("", encoding="utf-8")

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/tests/test_path_utils.py

@@ -30,7 +30,8 @@ def test_find_latest_incomplete_eval_results_path_picks_newest_matching(
 
     (old_run / "results.jsonl").write_text('{"example_id":0}\n', encoding="utf-8")
     (new_run / "results.jsonl").write_text(
-        '{"example_id":0}\n{"example_id":1}\n', encoding="utf-8"
+        '{"example_id":0}\n{"example_id":1}\n{"example_id":2',
+        encoding="utf-8",
     )
     (complete_run / "results.jsonl").write_text(
         '{"example_id":0}\n{"example_id":1}\n{"example_id":2}\n{"example_id":3}\n',

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/tests/test_save_utils.py

@@ -33,7 +33,6 @@ from verifiers.utils.save_utils import (
     save_new_outputs,
     save_metadata,
     states_to_outputs,
-    truncate_malformed_trailing_line,
     validate_resume_metadata,
 )
 from verifiers.utils.usage_utils import StateUsageTracker, response_usage_tokens
@@ -475,7 +474,7 @@ class TestSavingResults:
 
 
 class TestLoadOutputs:
-    def test_ignores_malformed_trailing_line(self, tmp_path: Path):
+    def test_ignores_malformed_trailing_line(self, tmp_path: Path, monkeypatch):
         results_path = tmp_path / "results"
         results_path.mkdir()
         outputs_path = results_path / "results.jsonl"
@@ -489,50 +488,68 @@ class TestLoadOutputs:
         outputs_path.write_text(
             "\n".join(lines + [partial_trailing_line]) + "\n", encoding="utf-8"
         )
+        warnings = []
+        monkeypatch.setattr(
+            "verifiers.utils.save_utils.logger.warning",
+            lambda *args, **kwargs: warnings.append(args),
+        )
 
         outputs = load_outputs(results_path)
 
-        assert len(outputs) == 2
-        assert outputs[0]["example_id"] == 0
-        assert outputs[1]["example_id"] == 1
+        assert [output["example_id"] for output in outputs] == [0, 1]
+        assert warnings
 
-    def test_raises_for_malformed_non_trailing_line(self, tmp_path: Path):
+    def test_ignores_malformed_non_trailing_line(self, tmp_path: Path, monkeypatch):
         results_path = tmp_path / "results"
         results_path.mkdir()
         outputs_path = results_path / "results.jsonl"
 
         malformed_non_trailing_line = '{"example_id": 0, "label": "broken"'
+        missing_example_id_line = json.dumps({"label": "missing"})
         valid_line = json.dumps({"example_id": 1, "label": "row-1"})
         outputs_path.write_text(
-            "\n".join([malformed_non_trailing_line, valid_line]) + "\n",
+            "\n".join(
+                [malformed_non_trailing_line, missing_example_id_line, valid_line]
+            )
+            + "\n",
             encoding="utf-8",
         )
+        warnings = []
+        monkeypatch.setattr(
+            "verifiers.utils.save_utils.logger.warning",
+            lambda *args, **kwargs: warnings.append(args),
+        )
 
-        with pytest.raises(json.JSONDecodeError):
-            load_outputs(results_path)
+        outputs = load_outputs(results_path)
+
+        assert [output["example_id"] for output in outputs] == [1]
+        assert warnings
 
 
 class TestSaveNewOutputs:
-    def test_truncates_malformed_trailing_line_before_append(self, tmp_path: Path):
+    def test_appends_after_malformed_rows_without_rewriting(self, tmp_path: Path):
         results_path = tmp_path / "results"
         results_path.mkdir()
         outputs_path = results_path / "results.jsonl"
 
-        existing_outputs = [
-            {"example_id": 0, "label": "row-0"},
-            {"example_id": 1, "label": "row-1"},
-        ]
+        malformed_middle_line = '{"example_id": 99, "label": "broken"'
         malformed_trailing_line = '{"example_id": 2, "label": "row-2"'
-        lines = [json.dumps(output) for output in existing_outputs]
         outputs_path.write_text(
-            "\n".join(lines + [malformed_trailing_line]), encoding="utf-8"
+            "\n".join(
+                [
+                    json.dumps({"example_id": 0, "label": "row-0"}),
+                    malformed_middle_line,
+                    json.dumps({"example_id": 1, "label": "row-1"}),
+                    malformed_trailing_line,
+                ]
+            ),
+            encoding="utf-8",
         )
 
-        # Caller drops the partial trailing row before appending so the new
-        # row lands on a valid JSONL boundary.
-        truncate_malformed_trailing_line(outputs_path)
+        circular_output = {"example_id": 4}
+        circular_output["self"] = circular_output
         save_new_outputs(
-            [{"example_id": 3, "label": "row-3"}],
+            [circular_output, {"example_id": 3, "label": "row-3"}],
             results_path,
         )
 
@@ -541,14 +558,12 @@ class TestSaveNewOutputs:
             for line in outputs_path.read_text(encoding="utf-8").splitlines()
             if line
         ]
-        parsed_outputs = [json.loads(line) for line in persisted_lines]
 
-        assert [output["example_id"] for output in parsed_outputs] == [0, 1, 3]
-        assert [output["example_id"] for output in load_outputs(results_path)] == [
-            0,
-            1,
-            3,
-        ]
+        assert persisted_lines[1] == malformed_middle_line
+        assert persisted_lines[3] == malformed_trailing_line
+        assert [
+            json.loads(persisted_lines[idx])["example_id"] for idx in [0, 2, 4]
+        ] == [0, 1, 3]
 
 
 class TestResumeMetadataValidation:

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/verifiers/envs/environment.py

@@ -82,7 +82,6 @@ from verifiers.utils.save_utils import (
     save_new_outputs,
     save_outputs,
     state_to_output,
-    truncate_malformed_trailing_line,
     validate_resume_metadata,
 )
 from verifiers.utils.usage_utils import StateUsageTracker
@@ -1003,9 +1002,21 @@ class Environment(ABC):
                 )
                 on_log(f"Resuming evaluation from {results_path}")
                 outputs = load_outputs(results_path)
-                # Drop any partial trailing row left by a crashed prior write
-                # so subsequent appends start from a valid JSONL boundary.
-                truncate_malformed_trailing_line(results_path / "results.jsonl")
+                rollout_counts_by_example_id: dict[object, int] = {}
+                capped_outputs: list[RolloutOutput] = []
+                for output in outputs:
+                    example_id = output["example_id"]
+                    rollout_count = rollout_counts_by_example_id.get(example_id, 0)
+                    if rollout_count >= rollouts_per_example:
+                        continue
+                    rollout_counts_by_example_id[example_id] = rollout_count + 1
+                    capped_outputs.append(output)
+                if len(capped_outputs) != len(outputs):
+                    on_log(
+                        f"Ignoring {len(outputs) - len(capped_outputs)} saved duplicate rollout(s) "
+                        "beyond rollouts_per_example"
+                    )
+                outputs = capped_outputs
                 builder.add_outputs(outputs)
                 filtered_inputs = filter_inputs(
                     raw_inputs, outputs, rollouts_per_example
@@ -1014,7 +1025,12 @@ class Environment(ABC):
                     on_log(
                         "No remaining rollouts to evaluate, returning completed outputs"
                     )
-                    return builder.build(sort_by_example_id=True)
+                    results = builder.build(sort_by_example_id=True)
+                    if save_results:
+                        await asyncio.to_thread(
+                            save_metadata, results["metadata"], builder.results_path
+                        )
+                    return results
                 on_log(
                     f"Found {len(outputs)} completed rollout(s), {len(filtered_inputs)} remaining rollout(s)"
                 )
@@ -1023,6 +1039,21 @@ class Environment(ABC):
 
             if save_results:
                 on_log(f"Saving results to {builder.results_path}")
+                if results_path is None or not is_valid_eval_results_path(results_path):
+                    outputs_path = builder.results_path / "results.jsonl"
+                    if (
+                        results_path is not None
+                        and outputs_path.is_file()
+                        and outputs_path.stat().st_size > 0
+                    ):
+                        raise ValueError(
+                            f"Cannot save to invalid results path {builder.results_path}: "
+                            "results.jsonl already exists without valid metadata"
+                        )
+                    await asyncio.to_thread(save_outputs, [], builder.results_path, "a")
+                    await asyncio.to_thread(
+                        save_metadata, builder.build_metadata(), builder.results_path
+                    )
 
             tasks: dict[asyncio.Task, int] = {}
             try:
@@ -1104,9 +1135,6 @@ class Environment(ABC):
 
             # save if requested
             if save_results:
-                await asyncio.to_thread(
-                    save_outputs, results["outputs"], builder.results_path
-                )
                 await asyncio.to_thread(
                     save_metadata, results["metadata"], builder.results_path
                 )

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/verifiers/utils/path_utils.py

@@ -81,25 +81,31 @@ def is_valid_eval_results_path(path: Path) -> bool:
     )
 
 
-def _count_saved_rollouts(results_path: Path) -> int:
+def _count_saved_rollouts(results_path: Path, rollouts_per_example: int) -> int:
     """Count completed rollout rows in results.jsonl."""
     outputs_path = results_path / "results.jsonl"
-    count = 0
-    with open(outputs_path, "r") as f:
+    counts: dict[object, int] = {}
+    with open(outputs_path, "rb") as f:
         for line_idx, line in enumerate(f, start=1):
             if not line.strip():
                 continue
             try:
-                json.loads(line)
-            except json.JSONDecodeError:
+                output = json.loads(line)
+            except (UnicodeDecodeError, json.JSONDecodeError):
                 logger.warning(
-                    "Ignoring malformed trailing line in %s at line %s",
+                    "Ignoring malformed JSONL row in %s at line %s",
                     outputs_path,
                     line_idx,
                 )
-                break
-            count += 1
-    return count
+                continue
+            if not isinstance(output, dict) or "example_id" not in output:
+                continue
+            example_id = output["example_id"]
+            counts[example_id] = min(
+                counts.get(example_id, 0) + 1,
+                rollouts_per_example,
+            )
+    return sum(counts.values())
 
 
 def find_latest_incomplete_eval_results_path(
@@ -160,7 +166,7 @@ def find_latest_incomplete_eval_results_path(
         if not isinstance(saved_num_examples, int) or saved_num_examples > num_examples:
             continue
 
-        saved_rollouts = _count_saved_rollouts(candidate)
+        saved_rollouts = _count_saved_rollouts(candidate, rollouts_per_example)
         if saved_rollouts < total_rollouts:
             return candidate
 

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/verifiers/utils/save_utils.py

@@ -1,5 +1,7 @@
 import json
 import logging
+import os
+import tempfile
 import time
 from collections import Counter
 from collections.abc import Mapping
@@ -709,30 +711,72 @@ def load_outputs(results_path: Path) -> list[RolloutOutput]:
     outputs_path = results_path / "results.jsonl"
     outputs: list[RolloutOutput] = []
 
-    with open(outputs_path, "r") as f:
-        lines = f.readlines()
+    with open(outputs_path, "rb") as f:
+        for line_idx, line in enumerate(f, start=1):
+            if not line.strip():
+                continue
+            try:
+                output = json.loads(line)
+            except (UnicodeDecodeError, json.JSONDecodeError):
+                logger.warning(
+                    "Ignoring malformed JSONL row in %s at line %s",
+                    outputs_path,
+                    line_idx,
+                )
+                continue
+            if not isinstance(output, dict):
+                logger.warning(
+                    "Ignoring non-object JSONL row in %s at line %s",
+                    outputs_path,
+                    line_idx,
+                )
+                continue
+            if "example_id" not in output:
+                logger.warning(
+                    "Ignoring JSONL row without example_id in %s at line %s",
+                    outputs_path,
+                    line_idx,
+                )
+                continue
+            outputs.append(RolloutOutput(**output))
 
-    for line_idx, line in enumerate(lines, start=1):
-        if not line.strip():
-            continue
+    return outputs
+
+
+def _append_prefix(outputs_path: Path) -> str:
+    """Add a newline if the existing file is missing one."""
+    if not outputs_path.exists() or outputs_path.stat().st_size == 0:
+        return ""
 
+    with open(outputs_path, "rb") as f:
+        f.seek(-1, 2)
+        if f.read(1) == b"\n":
+            return ""
+    return "\n"
+
+
+def save_outputs(
+    outputs: list[RolloutOutput], results_path: Path, mode: str = "a"
+) -> None:
+    """Save outputs to disk."""
+    serialized_rows: list[str] = []
+    for idx, output in enumerate(outputs):
+        example_id = output.get("example_id", "unknown")
         try:
-            outputs.append(RolloutOutput(**json.loads(line)))
-        except json.JSONDecodeError:
-            # A crash during append can leave the final JSONL line partially written.
-            # Recover completed records, but keep raising for malformed non-trailing rows.
-            has_nonempty_lines_after = any(
-                remaining.strip() for remaining in lines[line_idx:]
-            )
-            if has_nonempty_lines_after:
-                raise
+            serialized_rows.append(json.dumps(output, default=make_serializable) + "\n")
+        except Exception as e:
+            logger.error(f"Failed to save result with index {idx} ({example_id=}): {e}")
 
-            logger.warning(
-                f"Ignoring malformed trailing line in {outputs_path} at line {line_idx}"
-            )
-            break
+    serialized = "".join(serialized_rows)
 
-    return outputs
+    results_path.mkdir(parents=True, exist_ok=True)
+    outputs_path = results_path / "results.jsonl"
+    prefix = ""
+    if "a" in mode and serialized:
+        prefix = _append_prefix(outputs_path)
+
+    with open(outputs_path, mode) as f:
+        f.write(prefix + serialized)
 
 
 def validate_resume_metadata(
@@ -807,77 +851,9 @@ def validate_resume_metadata(
         )
 
 
-def save_outputs(outputs: list[RolloutOutput], results_path: Path, mode: str = "w"):
-    """Save outputs to disk."""
-    results_path.mkdir(parents=True, exist_ok=True)
-    outputs_path = results_path / "results.jsonl"
-    with open(outputs_path, mode) as f:
-        for idx, output in enumerate(outputs):
-            example_id = output.get("example_id") or "unknown"
-            try:
-                json.dump(output, f, default=make_serializable)
-                f.write("\n")
-            except Exception as e:
-                logger.error(
-                    f"Failed to save result with index {idx} ({example_id=}): {e}"
-                )
-
-
-def _get_last_nonempty_line_bounds(file_obj: Any) -> tuple[int, bytes] | None:
-    """Return byte offset + contents for the last non-empty line in a file."""
-    file_obj.seek(0, 2)
-    file_size = file_obj.tell()
-    if file_size == 0:
-        return None
-
-    cursor = file_size
-
-    # Skip trailing whitespace/newlines to locate the real end of the last row.
-    while cursor > 0:
-        cursor -= 1
-        file_obj.seek(cursor)
-        if file_obj.read(1) not in b" \t\r\n":
-            break
-    else:
-        return None
-
-    line_end = cursor + 1
-    line_start = cursor
-    while line_start > 0:
-        file_obj.seek(line_start - 1)
-        if file_obj.read(1) == b"\n":
-            break
-        line_start -= 1
-
-    file_obj.seek(line_start)
-    return line_start, file_obj.read(line_end - line_start)
-
-
-def truncate_malformed_trailing_line(outputs_path: Path) -> None:
-    """Drop a malformed trailing JSONL row so future appends stay valid."""
-    if not outputs_path.exists() or not outputs_path.is_file():
-        return
-
-    with open(outputs_path, "rb+") as f:
-        last_line_info = _get_last_nonempty_line_bounds(f)
-        if last_line_info is None:
-            return
-
-        line_start, line_bytes = last_line_info
-        try:
-            json.loads(line_bytes.decode("utf-8"))
-        except (UnicodeDecodeError, json.JSONDecodeError):
-            logger.warning(
-                "Removing malformed trailing line in %s at byte offset %s",
-                outputs_path,
-                line_start,
-            )
-            f.truncate(line_start)
-
-
 def save_new_outputs(new_outputs: list[RolloutOutput], results_path: Path):
     """Saves new rollout outputs to disk (in append mode)."""
-    save_outputs(new_outputs, results_path, mode="a")
+    save_outputs(new_outputs, results_path)
 
 
 def save_metadata(metadata: GenerateMetadata, result_path: Path):
@@ -888,11 +864,16 @@ def save_metadata(metadata: GenerateMetadata, result_path: Path):
     metadata_dict = dict(metadata)
     metadata_dict.pop("path_to_save")
     metadata_dict.pop("date")
-    with open(metadata_path, "w") as f:
-        try:
-            json.dump(metadata_dict, f, default=make_serializable)
-        except Exception as e:
-            logger.error(f"Failed to save metadata: {e}")
+    serialized = json.dumps(metadata_dict, default=make_serializable)
+    temp_path = None
+    try:
+        with tempfile.NamedTemporaryFile("w", dir=result_path, delete=False) as f:
+            temp_path = f.name
+            f.write(serialized)
+        os.replace(temp_path, metadata_path)
+    finally:
+        if temp_path is not None and os.path.exists(temp_path):
+            os.unlink(temp_path)
 
 
 def make_dataset(results: GenerateOutputs) -> Dataset:

{verifiers-0.1.15.dev171 → verifiers-0.1.15.dev173}/verifiers/v1/utils/sandbox_utils.py

@@ -44,7 +44,7 @@ if TYPE_CHECKING:
 
 VF_STATE_INPUT_PATH_KEY = "_vf_state_input_path"
 SANDBOX_RETRY_ATTEMPTS = 6
-SANDBOX_WAIT_FOR_CREATION_ATTEMPTS = 120
+SANDBOX_WAIT_FOR_CREATION_ATTEMPTS = 360
 SANDBOX_TRANSFER_RETRY_TOKENS = (
     "502",
     "503",