verifiers 0.4.1.dev34__tar.gz → 0.4.1.dev35__tar.gz

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev34
+Version: 0.4.1.dev35
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev35}/verifiers/v1/tasksets/harbor/taskset.py

@@ -830,12 +830,10 @@ def parse_verifier_extras(
 
     artifacts: list[HarborArtifact] = []
     for entry in normalize_artifact_entries(parsed.artifacts):
-        # `destination` positions a file in Harbor's host trial directory. Verifiers has
-        # no such directory (the trace is the record) and Harbor never lets destination
-        # affect verifier-side placement, so it cannot change any grading outcome.
         artifacts.append(
             HarborArtifact(
                 source=entry.source,
+                destination=entry.destination,
                 service=effective_artifact_service(entry),
                 exclude=list(entry.exclude or []),
                 required=False,

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev35}/verifiers/v1/utils/artifacts.py

@@ -3,14 +3,17 @@
 from __future__ import annotations
 
 import io
+import json
 import logging
 import shlex
+import shutil
 import tarfile
+import tempfile
 import uuid
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 if TYPE_CHECKING:
     from verifiers.v1.runtimes import Runtime
@@ -29,10 +32,31 @@ class Artifact(BaseModel):
     """One path to restore at the same location in another runtime."""
 
     source: str
+    destination: str | None = None
+    """Saved path relative to the trace's artifact directory; grading uses `source`."""
     exclude: list[str] = Field(default_factory=list)
     """`tar --exclude` patterns, applied when `source` is a directory."""
     required: bool = True
 
+    @field_validator("destination")
+    @classmethod
+    def validate_destination(cls, value: str | None) -> str | None:
+        if not value:
+            return None
+        path = PurePosixPath(value)
+        if (
+            path.is_absolute()
+            or not path.parts
+            or ".." in path.parts
+            or "\\" in value
+            or path.parts[0] == "manifest.json"
+        ):
+            raise ValueError(
+                "artifact destination must be a relative path without '..' or "
+                "backslashes, outside the reserved manifest.json path"
+            )
+        return str(path)
+
 
 async def collect(
     runtime: Runtime,
@@ -136,6 +160,100 @@ async def collect(
     return collected
 
 
+def save_artifacts(
+    directory: Path,
+    collected: dict[str, bytes | None],
+    artifacts: list[Artifact],
+    *,
+    workdir: str = "/",
+) -> None:
+    """Save collected files at their declared destinations, with a manifest.
+
+    Unpack each validated archive in isolation before moving its files into the
+    saved tree. Failed or conflicting entries are recorded without losing the
+    completed trace. Source-mirrored directories may merge: collection splits
+    nested declarations out of the implicit convention sweep.
+    """
+    declared = {str(PurePosixPath(workdir) / a.source): a for a in artifacts}
+    manifest: list[dict] = []
+    claims: list[tuple[PurePosixPath, bool]] = []
+    directory.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(dir=directory.parent) as temporary:
+        saved = Path(temporary) / "artifacts"
+        saved.mkdir()
+        (saved / "manifest.json").touch()
+        for source, archive in collected.items():
+            artifact = declared.get(source, Artifact(source=source))
+            destination = artifact.destination or artifact.source.lstrip("/")
+            entry = {
+                "source": source,
+                "service": getattr(artifact, "service", "main"),
+                "destination": destination,
+                "status": "empty",
+            }
+            manifest.append(entry)
+            try:
+                _validate_restore(source, archive)
+                Artifact.validate_destination(artifact.destination)
+                target = PurePosixPath(destination)
+                entry["destination"] = str(target)
+                if archive is None:
+                    continue
+                with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar:
+                    members = tar.getmembers()
+                    if not members:
+                        continue
+                    root = PurePosixPath(source).relative_to("/")
+                    paths = [
+                        saved / target / PurePosixPath(member.name).relative_to(root)
+                        for member in members
+                    ]
+                    # Explicit renames claim a whole subtree. Source-mirrored
+                    # entries may share directories, but never overwrite files.
+                    if any(
+                        (artifact.destination or renamed)
+                        and (
+                            target.is_relative_to(prior) or prior.is_relative_to(target)
+                        )
+                        for prior, renamed in claims
+                    ) or any(
+                        path.is_symlink()
+                        or (path.exists() and not (member.isdir() and path.is_dir()))
+                        or any(
+                            parent.is_symlink()
+                            or (parent.exists() and not parent.is_dir())
+                            for parent in path.parents
+                            if parent.is_relative_to(saved)
+                        )
+                        for member, path in zip(members, paths, strict=True)
+                    ):
+                        entry["status"] = "skipped"
+                        entry["error"] = (
+                            "destination overlaps a previously saved artifact"
+                        )
+                        logger.warning("artifact %r: %s", source, entry["error"])
+                        continue
+                    with tempfile.TemporaryDirectory(dir=temporary) as unpacked:
+                        tar.extractall(unpacked, filter="data")
+                        for member, path in zip(members, paths, strict=True):
+                            if member.isdir():
+                                path.mkdir(parents=True, exist_ok=True)
+                            else:
+                                path.parent.mkdir(parents=True, exist_ok=True)
+                                (Path(unpacked) / member.name).replace(path)
+                    claims.append((target, artifact.destination is not None))
+                    entry["status"] = "ok"
+            except (OSError, ValueError, RuntimeError, tarfile.TarError) as exc:
+                entry.update(status="failed", error=str(exc))
+                logger.warning("failed to save artifact %r: %s", source, exc)
+        (saved / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+        if directory.is_symlink():
+            raise ValueError(f"artifact directory must not be a symlink: {directory}")
+        if directory.exists():
+            shutil.rmtree(directory)
+        saved.replace(directory)
+
+
 async def restore(runtime: Runtime, collected: dict[str, bytes | None]) -> None:
     """Extract `collected` in `runtime` at the original absolute paths.
 

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev35}/verifiers/v1/utils/trace_store.py

@@ -14,6 +14,7 @@ from verifiers.v1.state import StateT
 from verifiers.v1.task import DataT
 from verifiers.v1.trace import AgentConfigT, Trace
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.artifacts import save_artifacts
 
 TRACES_FILE = "traces.jsonl"
 type_adapter = cache(TypeAdapter)
@@ -25,6 +26,18 @@ def write_episode(
     """Serialize and append one rollout episode in the worker thread."""
     # Preserve fields declared by typed Trace subclasses nested in the episode.
     data = type_adapter(type(episode)).dump_json(episode, exclude_none=True)
+    for trace in episode.traces:
+        if trace.state.artifacts:
+            if trace.id == ".." or Path(trace.id).parts != (trace.id,):
+                raise ValueError(
+                    f"trace ID must be a single path component: {trace.id!r}"
+                )
+            save_artifacts(
+                results_dir / "artifacts" / trace.id,
+                trace.state.artifacts,
+                trace.task.data.artifacts,
+                workdir=getattr(trace.agent.runtime, "workdir", None) or "/",
+            )
     with (results_dir / TRACES_FILE).open("ab") as f:
         f.write(data + b"\n")