verifiers 0.4.2.dev27__tar.gz → 0.4.2.dev29__tar.gz

{verifiers-0.4.2.dev27 → verifiers-0.4.2.dev29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.2.dev27
+Version: 0.4.2.dev29
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.2.dev27 → verifiers-0.4.2.dev29}/tests/unit/test_taskset.py

@@ -102,6 +102,16 @@ def test_exclude_drops_tasks_that_any_list_names() -> None:
     assert idxs(finite().exclude(idx=":7", names=["t8"])) == [7, 9]
 
 
+def test_match_lists_extend_from_files(tmp_path) -> None:
+    (tmp_path / "idx.txt").write_text("0:2\n\n5\n")
+    (tmp_path / "names.txt").write_text(" t8 \nt9\n")
+    files = {"idx": tmp_path / "idx.txt", "names": tmp_path / "names.txt"}
+    assert idxs(finite().include(idx=[3], files=files)) == [0, 1, 3, 5, 8, 9]
+    config = vf.SelectConfig(exclude={"files": {"names": tmp_path / "names.txt"}})
+    assert config.exclude.names == ["t8", "t9"] and config.exclude.files.names is None
+    assert idxs(finite().select(config)) == list(range(8))
+
+
 def test_unmatched_entries_warn(caplog) -> None:
     with caplog.at_level(logging.WARNING):
         assert idxs(finite().include(names=["t1", "typo"], idx=[42])) == [1]

verifiers-0.4.2.dev29/tests/unit/utils/test_artifacts.py

@@ -0,0 +1,114 @@
+"""Artifact restore: the grading box extracts only what host-side validation read."""
+
+import asyncio
+import io
+import shutil
+import subprocess
+import tarfile
+from types import SimpleNamespace
+
+import pytest
+
+from verifiers.v1.utils.artifacts import restore
+
+
+class RecordingRuntime:
+    """Records the archives `restore` sends and accepts every command."""
+
+    def __init__(self) -> None:
+        self.config = SimpleNamespace(type="docker", mounts={}, workdir="/app")
+        self.written: list[bytes] = []
+
+    async def write(self, path: str, data: bytes) -> None:
+        self.written.append(data)
+
+    async def run(self, argv, env):
+        return SimpleNamespace(exit_code=0, stdout="", stderr="")
+
+
+def make_archive(build) -> bytes:
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w:", format=tarfile.GNU_FORMAT) as tar:
+        build(tar)
+    return buffer.getvalue()
+
+
+def add_file(tar: tarfile.TarFile, name: str, data: bytes, **fields) -> None:
+    info = tarfile.TarInfo(name)
+    info.size = len(data)
+    for key, value in fields.items():
+        setattr(info, key, value)
+    tar.addfile(info, io.BytesIO(data))
+
+
+def restored(archive: bytes, root: str = "/app/results") -> bytes:
+    runtime = RecordingRuntime()
+    asyncio.run(restore(runtime, {root: archive}))
+    assert len(runtime.written) == 1
+    return runtime.written[0]
+
+
+def test_restore_keeps_owners_and_drops_special_mode_bits() -> None:
+    def build(tar):
+        directory = tarfile.TarInfo("app/results")
+        directory.type, directory.mode, directory.uid = tarfile.DIRTYPE, 0o1777, 1000
+        tar.addfile(directory)
+        add_file(
+            tar, "app/results/run", b"#!/bin/sh\n", mode=0o4755, uid=1000, gid=1000
+        )
+
+    with tarfile.open(fileobj=io.BytesIO(restored(make_archive(build)))) as tar:
+        members = {m.name: m for m in tar.getmembers()}
+
+    assert members["app/results"].mode == 0o777
+    assert members["app/results/run"].mode == 0o755
+    assert (members["app/results/run"].uid, members["app/results/run"].gid) == (
+        1000,
+        1000,
+    )
+
+
+@pytest.mark.skipif(shutil.which("tar") is None, reason="needs a system tar")
+def test_system_tar_lists_exactly_what_python_read() -> None:
+    long_name = "app/results/" + "nested/" * 20 + "phase_map.csv"
+
+    def build(tar):
+        add_file(tar, "app/results/a.txt", b"a")
+        add_file(tar, long_name, b"b")
+        link = tarfile.TarInfo("app/results/link")
+        link.type, link.linkname = tarfile.SYMTYPE, "a.txt"
+        tar.addfile(link)
+        hard = tarfile.TarInfo("app/results/hard")
+        hard.type, hard.linkname = tarfile.LNKTYPE, "app/results/a.txt"
+        tar.addfile(hard)
+
+    original = make_archive(build)
+    with tarfile.open(fileobj=io.BytesIO(original)) as tar:
+        python_names = [m.name for m in tar.getmembers()]
+
+    listing = subprocess.run(
+        ["tar", "-tf", "-"],
+        input=restored(original),
+        capture_output=True,
+        check=True,
+    ).stdout.decode()
+
+    assert listing.splitlines() == python_names
+
+
+def test_restore_refuses_sparse_members() -> None:
+    # A few header bytes declare a terabyte: rebuilding it would fill host memory.
+    header = tarfile.TarInfo("app/results/hole")
+    header.type = tarfile.GNUTYPE_SPARSE
+    block = bytearray(header.tobuf(format=tarfile.GNU_FORMAT))
+    block[386:398] = b"00000000000\0"  # first sparse entry: offset 0
+    block[398:410] = b"00000000000\0"  # ... holding 0 bytes of data
+    block[483:495] = b"%011o\0" % (1 << 40)  # real size
+    block[148:156] = b" " * 8
+    block[148:156] = b"%06o\0 " % sum(block)
+    archive = bytes(block) + bytes(2 * tarfile.BLOCKSIZE)
+
+    runtime = RecordingRuntime()
+    with pytest.raises(RuntimeError, match="sparse"):
+        asyncio.run(restore(runtime, {"/app/results": archive}))
+    assert runtime.written == []

{verifiers-0.4.2.dev27 → verifiers-0.4.2.dev29}/verifiers/v1/configs/select.py

@@ -3,13 +3,24 @@
 
 import re
 import sys
+from pathlib import Path
 
-from pydantic import AliasChoices, Field, field_validator
+from pydantic import AliasChoices, Field, field_validator, model_validator
 from pydantic_config import BaseConfig
 
 IDX_RANGE = re.compile(r"(\d*):(\d*)(?::(\d*))?")
 
 
+class TaskMatchFilesConfig(BaseConfig):
+    """Newline-delimited text files with more entries for each `TaskMatchConfig`
+    list, one entry per line. Blank lines are skipped."""
+
+    idx: Path | None = None
+    ids: Path | None = None
+    keys: Path | None = None
+    names: Path | None = None
+
+
 class TaskMatchConfig(BaseConfig):
     """Tasks named by position or identity; a task matches when any list names it."""
 
@@ -24,6 +35,10 @@ class TaskMatchConfig(BaseConfig):
     """`Task.key` values, as recorded on each trace (`task.key`)."""
     names: list[str] = Field(default_factory=list)
     """`TaskData.name` values."""
+    files: TaskMatchFilesConfig = TaskMatchFilesConfig()
+    """Files whose lines extend the lists above, for selections too long to write
+    inline (`files.ids = "ids.txt"`). Validation reads them into the lists and
+    resets `files`."""
 
     @field_validator("idx", mode="before")
     @classmethod
@@ -36,6 +51,19 @@ class TaskMatchConfig(BaseConfig):
             items.extend(_parse_idx_item(part) for part in parts)
         return items
 
+    @model_validator(mode="after")
+    def _read_files(self):
+        for field, path in self.files:
+            if path is None:
+                continue
+            lines = [line.strip() for line in path.read_text().splitlines()]
+            entries = [line for line in lines if line]
+            if field == "idx":
+                entries = [_parse_idx_item(entry) for entry in entries]
+            getattr(self, field).extend(entries)
+        self.files = TaskMatchFilesConfig()
+        return self
+
     @property
     def empty(self) -> bool:
         return not (self.idx or self.ids or self.keys or self.names)

{verifiers-0.4.2.dev27 → verifiers-0.4.2.dev29}/verifiers/v1/tasksets/harbor/taskset.py

@@ -410,7 +410,10 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
                 "mkdir -p /tests && tar -xzf /tmp/tests.tgz -C /tests"
             )
         # Verifier subprocesses can use different users; opening their log directory
-        # requires its owner or root, even when it is already writable.
+        # requires its owner or root, even when it is already writable. A grading box
+        # of its own starts with an empty log directory: anything already there came
+        # from the image or from the agent's restored artifacts, not from this grader.
+        isolated = wipe or self.scoring.mode == "isolated"
         setup = (
             runtime.with_user("root")
             if runtime.backend.SUPPORTS_USER_SWITCHING
@@ -420,6 +423,7 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
             (
                 setup,
                 (
+                    f"{'rm -rf /logs/verifier && ' if isolated else ''}"
                     f"mkdir -p /logs/verifier {ARTIFACTS_DIR} && "
                     f"chmod 777 /logs/verifier {ARTIFACTS_DIR}"
                 ),

{verifiers-0.4.2.dev27 → verifiers-0.4.2.dev29}/verifiers/v1/utils/artifacts.py

@@ -341,6 +341,10 @@ async def restore(runtime: Runtime, collected: dict[str, bytes | None]) -> None:
         )
     for root, archive in collected.items():
         _validate_restore(root, archive)
+    collected = {
+        root: None if archive is None else _rebuild(archive)
+        for root, archive in collected.items()
+    }
     # Clear every root up front, not per entry: a later nested root would otherwise
     # delete content an earlier one just restored. Clearing also drops any file or
     # symlink the image left at the target.
@@ -425,6 +429,45 @@ def _drop_special_files(archive: bytes) -> tuple[bytes, list[str]]:
     return out.getvalue(), dropped
 
 
+def _rebuild(archive: bytes) -> bytes:
+    """Write a fresh archive holding exactly the members Python's tarfile reads.
+
+    Validation reads archives with Python's tarfile, but the grading box extracts
+    them with its own tar. An agent controls the archive bytes, and two tar parsers
+    can disagree about the same bytes, so the box only ever extracts an archive
+    written here from the members validation saw. Owners and permission bits are
+    kept (tasks rely on ownership, e.g. git's safe.directory check); setuid,
+    setgid and sticky bits are not.
+    """
+    out = io.BytesIO()
+    with (
+        tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as source,
+        tarfile.open(fileobj=out, mode="w:", format=tarfile.PAX_FORMAT) as target,
+    ):
+        for member in source:
+            info = tarfile.TarInfo(member.name)
+            info.mode = member.mode & 0o777
+            info.uid, info.gid = member.uid, member.gid
+            info.uname, info.gname = member.uname, member.gname
+            info.mtime = member.mtime
+            content = None
+            if member.isfile():
+                info.type = tarfile.REGTYPE
+                info.size = member.size
+                content = source.extractfile(member)
+            elif member.isdir():
+                info.type = tarfile.DIRTYPE
+            elif member.issym() or member.islnk():
+                info.type = tarfile.SYMTYPE if member.issym() else tarfile.LNKTYPE
+                info.linkname = member.linkname
+            else:
+                raise RuntimeError(
+                    f"artifact member {member.name!r} is a link or special file"
+                )
+            target.addfile(info, content)
+    return out.getvalue()
+
+
 def _validate_restore(root: str, archive: bytes | None) -> None:
     """Reject archive content that could restore outside its declared root."""
     root_path = PurePosixPath(root)
@@ -482,6 +525,15 @@ def _validate_restore(root: str, archive: bytes | None) -> None:
                     raise RuntimeError(
                         f"artifact member {member.name!r} is a link or special file"
                     )
+                if member.issparse():
+                    # A sparse member declares its full size in a few header bytes,
+                    # and rebuilding the archive on the host would write every hole
+                    # out as zeros. GNU tar only writes these with --sparse; bsdtar
+                    # writes them for any file with holes.
+                    raise RuntimeError(
+                        f"artifact member {member.name!r} is a sparse file, which "
+                        "cannot be restored"
+                    )
                 if member.isfile():
                     regular_files.add(member_path)
             # Check after reading the whole archive, so a later symlink cannot