verifiers 0.4.1.dev34__tar.gz → 0.4.1.dev36__tar.gz

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev34
+Version: 0.4.1.dev36
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev36}/tests/integration/test_multi_turn_tool.py

@@ -28,7 +28,11 @@ async def test_tool(run, harness, harness_runtime):
     # Reward 1.0 is the proof the tool ran. How a program surfaces an MCP tool to the
     # model is its own business (rlm calls it from code; chat loops advertise it), so
     # the advertised schema is checked on the chat loop in `test_tool_placement`.
-    assert trace.reward == 1.0
+    assert trace.reward == 1.0, "\n".join(
+        message.model_dump_json(exclude_none=True)
+        for message in trace.messages
+        if message.role in {"assistant", "tool"}
+    )
 
 
 @pytest.mark.parametrize(

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev36}/verifiers/v1/harnesses/prime_agent/harness.py

@@ -8,10 +8,10 @@ from typing import Literal
 from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig, skill_destination
-from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
 from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
 from verifiers.v1.harnesses.utils.replay import drop_blank_text
 from verifiers.v1.runtimes import Runtime
+from verifiers.v1.runtimes.base import _ENSURE_UV
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
 from verifiers.v1.types import Message
@@ -21,10 +21,10 @@ logger = logging.getLogger(__name__)
 GITHUB_RELEASE_URL = (
     "https://github.com/PrimeIntellect-ai/prime-agent/releases/download"
 )
-PRIME_AGENT_COMMIT: Literal["a7d791bc1be09793ed5f3ec05bf4cccbc60679ea"] = (
-    "a7d791bc1be09793ed5f3ec05bf4cccbc60679ea"
+PRIME_AGENT_COMMIT: Literal["763094e1b6a7c17450ee8399b06cda1853847ca5"] = (
+    "763094e1b6a7c17450ee8399b06cda1853847ca5"
 )
-PRIME_AGENT_VERSION = "0.9.5"
+PRIME_AGENT_VERSION = "0.10.0"
 PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent"
 STATE_ROOT = "/tmp/vf-prime-agent-runs"
 PROVIDER = "intercept"
@@ -33,63 +33,51 @@ KEY_VAR = "PRIME_AGENT_INTERCEPT_KEY"
 ENV_AGENT_DIR = "PRIME_AGENT_CODING_AGENT_DIR"
 
 
-INSTALL = r"""
-set -e
-export PATH="/var/tmp/vf-node/bin:$PATH"
+INSTALL = (
+    "set -e\n"
+    + _ENSURE_UV
+    + r"""
 prefix="$VF_PRIME_AGENT_DIR/$PRIME_AGENT_COMMIT"
-[ -x "$prefix/bin/prime-agent" ] && [ -f "$PRIME_AGENT_KERNEL_VENV/.bootstrap-version" ] && exit 0
-export NPM_CONFIG_PREFIX="$prefix"
-export PRIME_AGENT_BOOTSTRAP_KERNEL_ON_INSTALL=1
-release_url="$VF_PRIME_AGENT_GITHUB_RELEASE_URL/v$PRIME_AGENT_RELEASE_VERSION"
-agent_tarball="prime-agent-$PRIME_AGENT_RELEASE_VERSION.tgz"
-ai_tarball="prime-agent-ai-$PRIME_AGENT_RELEASE_VERSION.tgz"
-core_tarball="prime-agent-core-$PRIME_AGENT_RELEASE_VERSION.tgz"
-tui_tarball="prime-agent-tui-$PRIME_AGENT_RELEASE_VERSION.tgz"
-download_dir="$(mktemp -d "$VF_PRIME_AGENT_DIR/install.XXXXXX")"
-trap 'rm -rf "$download_dir"' EXIT
-for tarball in "$agent_tarball" "$ai_tarball" "$core_tarball" "$tui_tarball"; do
-    curl -fsSL --retry 5 --retry-all-errors \
-        "$release_url/$tarball" -o "$download_dir/$tarball"
+if [ ! -x "$prefix/bin/prime-agent" ]; then
+    case "$(uname -s)-$(uname -m)" in
+        Linux-x86_64) platform=linux-x64; sha=c16bd2af5e77b53f49b914a44742c4cf6a67c5ed5000041430b78dbd4c3bcbec ;;
+        Linux-aarch64|Linux-arm64) platform=linux-arm64; sha=f3cab3530a4d7ca43dbef8321bf13f260d1ba05d8b5dde057165a20bd4f675c4 ;;
+        Darwin-x86_64) platform=darwin-x64; sha=af4866b5ba82f3419b964290e3f023ddb9b99958a89e16ee856962b901ec0fc6 ;;
+        Darwin-arm64) platform=darwin-arm64; sha=e418bdf62fcb0002777bf3ac43bcc136512b26763f2ab5c500792f26e37294e5 ;;
+        *) echo "unsupported prime-agent platform: $(uname -s)-$(uname -m)" >&2; exit 1 ;;
+    esac
+    command -v curl >/dev/null || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } || apk add --no-cache curl ca-certificates
+    release_url="$VF_PRIME_AGENT_GITHUB_RELEASE_URL/v$PRIME_AGENT_RELEASE_VERSION"
+    tarball="prime-agent-$PRIME_AGENT_RELEASE_VERSION-$platform.tar.gz"
+    download_dir="$(mktemp -d "$VF_PRIME_AGENT_DIR/install.XXXXXX")"
+    trap 'rm -rf "$download_dir"' EXIT
+    curl -fsSL --retry 5 --retry-all-errors "$release_url/$tarball" -o "$download_dir/$tarball"
+    printf '%s  %s\n' "$sha" "$tarball" > "$download_dir/SHA256SUMS"
+    if command -v sha256sum >/dev/null; then
+        (cd "$download_dir" && sha256sum -c SHA256SUMS)
+    else
+        (cd "$download_dir" && shasum -a 256 -c SHA256SUMS)
+    fi
+    mkdir "$download_dir/bin"
+    tar -xzf "$download_dir/$tarball" -C "$download_dir/bin"
+    [ "$("$download_dir/bin/prime-agent" --version)" = "$PRIME_AGENT_RELEASE_VERSION" ]
+    mkdir -p "$prefix"
+    mv "$download_dir/bin" "$prefix/bin"
+fi
+# Let the released binary validate the kernel cache and prepare its Python runtime.
+"$prefix/bin/prime-agent" --prime-agent-bootstrap
+# Bootstrap installs only the runtime; sessions also import the bundled Python skills.
+set --
+for pyproject in "$prefix/bin/skills/"*/pyproject.toml; do
+    set -- "$@" --editable "${pyproject%/pyproject.toml}"
 done
-printf '%s  %s\n' \
-    '349f1682c7909550842f1b04a71ba95814341b136474ade736df93f8ec006876' "$agent_tarball" \
-    '9ad0184b7b5f3d5c3b1677c0663335de32c092cbdb72697aeaf997f17910bb92' "$ai_tarball" \
-    '59aafeffb4b64eb997399d56a03f529b2f6604b228e5674372195277db52f04a' "$core_tarball" \
-    'e49f41170edfd1d0e72418729d2c117969ccf8df784dc55fe8c1a9d91778e103' "$tui_tarball" \
-    > "$download_dir/SHA256SUMS"
-(cd "$download_dir" && sha256sum -c SHA256SUMS)
-mkdir "$download_dir/package-root"
-tar -xzf "$download_dir/$agent_tarball" -C "$download_dir/package-root"
-node - \
-    "$download_dir/package-root/package/package.json" \
-    "$download_dir" \
-    "$ai_tarball" \
-    "$core_tarball" \
-    "$tui_tarball" <<'NODE'
-const fs = require("node:fs");
-const [manifestPath, downloadDir, ai, core, tui] = process.argv.slice(2);
-const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
-for (const [name, file] of [
-    ["@earendil-works/pi-ai", ai],
-    ["@earendil-works/pi-agent-core", core],
-    ["@earendil-works/pi-tui", tui],
-]) {
-    manifest.dependencies[name] = `file:${downloadDir}/${file}`;
-}
-fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
-NODE
-mkdir "$download_dir/repacked"
-repacked="$(npm pack "$download_dir/package-root/package" \
-    --pack-destination "$download_dir/repacked" --silent)"
-PRIME_AGENT_BOOTSTRAP_TOOLS_ON_INSTALL=1 npm install -g \
-    --no-fund --no-audit --loglevel=error --progress=false \
-    "$download_dir/repacked/$repacked"
-[ -f "$PRIME_AGENT_KERNEL_VENV/.bootstrap-version" ]
+uv pip install --python "$PRIME_AGENT_KERNEL_VENV/bin/python" "$@"
 """
+)
 
 
 class PrimeAgentHarnessConfig(HarnessConfig):
-    commit: Literal["a7d791bc1be09793ed5f3ec05bf4cccbc60679ea"] = PRIME_AGENT_COMMIT
+    commit: Literal["763094e1b6a7c17450ee8399b06cda1853847ca5"] = PRIME_AGENT_COMMIT
     """Prime Agent release commit to install."""
 
     autonomous: bool = False
@@ -101,8 +89,8 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
     SUPPORTS_MCP = True
     SUPPORTS_SKILLS = True
 
-    # Its pi-ai fork drops whitespace-only assistant text on replay. Recheck on bumps:
-    # https://github.com/PrimeIntellect-ai/prime-agent/blob/v0.9.5/packages/ai/src/providers/openai-completions.ts#L895
+    # Its chat-completions provider drops whitespace-only assistant text on replay. Recheck on bumps:
+    # https://github.com/PrimeIntellect-ai/prime-agent/blob/v0.10.0/crates/pa-ai/src/providers/openai_completions/convert.rs#L106
     def replay_view(self, message: Message) -> Message:
         return drop_blank_text(message)
 
@@ -161,7 +149,6 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
         statuses.append(status)
 
     async def setup(self, runtime: Runtime) -> None:
-        await ensure_node(runtime)
         logger.info("prime-agent: ensuring commit %s is installed", self.config.commit)
         await ensure_installed(
             runtime,
@@ -262,6 +249,18 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
             args += ["--skill", skill_destination(skill, skills_dir)]
         if system_prompt:
             args += ["--append-system-prompt", system_prompt]
+        if mcp_urls:
+            # ACP servers need discovery guidance; importing the SDK shadows the REPL helper.
+            args += [
+                "--append-system-prompt",
+                (
+                    f"Connected MCP servers: {json.dumps(list(mcp_urls))}. "
+                    "Use the pre-imported `mcp` helper in `ipython`, or `from rlm import mcp`. "
+                    "Do not `import mcp`: that loads the unrelated SDK and overwrites the helper. "
+                    "Discover tools with `print(await mcp.list_tools(server))`, then call them "
+                    "with `print(await mcp.call_tool(server, tool, arguments))`."
+                ),
+            ]
 
         return ACPConfig(
             env=self._env(trace, secret),
@@ -270,7 +269,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
                 "/bin/sh",
                 "-eu",
                 "-c",
-                f'export PATH="{NODE_BIN_DIR}:$HOME/.local/bin:$PATH"; exec "$@"',
+                'export PATH="$HOME/.local/bin:$PATH"; exec "$@"',
                 "prime-agent",
                 *args,
             ],
@@ -301,6 +300,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
             KEY_VAR: secret,
             ENV_AGENT_DIR: f"{root}/agent",
             "TMPDIR": f"{root}/tmp",
-            "PRIME_AGENT_KERNEL_VENV": self._kernel_venv(),
+            # Sessions use the installed interpreter without writing to the shared venv.
+            "PRIME_AGENT_KERNEL_PYTHON": f"{self._kernel_venv()}/bin/python",
             "PRIME_AGENT_TELEMETRY": "0",
         }

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev36}/verifiers/v1/tasksets/harbor/taskset.py

@@ -830,12 +830,10 @@ def parse_verifier_extras(
 
     artifacts: list[HarborArtifact] = []
     for entry in normalize_artifact_entries(parsed.artifacts):
-        # `destination` positions a file in Harbor's host trial directory. Verifiers has
-        # no such directory (the trace is the record) and Harbor never lets destination
-        # affect verifier-side placement, so it cannot change any grading outcome.
         artifacts.append(
             HarborArtifact(
                 source=entry.source,
+                destination=entry.destination,
                 service=effective_artifact_service(entry),
                 exclude=list(entry.exclude or []),
                 required=False,

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev36}/verifiers/v1/utils/artifacts.py

@@ -3,14 +3,17 @@
 from __future__ import annotations
 
 import io
+import json
 import logging
 import shlex
+import shutil
 import tarfile
+import tempfile
 import uuid
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 if TYPE_CHECKING:
     from verifiers.v1.runtimes import Runtime
@@ -29,10 +32,31 @@ class Artifact(BaseModel):
     """One path to restore at the same location in another runtime."""
 
     source: str
+    destination: str | None = None
+    """Saved path relative to the trace's artifact directory; grading uses `source`."""
     exclude: list[str] = Field(default_factory=list)
     """`tar --exclude` patterns, applied when `source` is a directory."""
     required: bool = True
 
+    @field_validator("destination")
+    @classmethod
+    def validate_destination(cls, value: str | None) -> str | None:
+        if not value:
+            return None
+        path = PurePosixPath(value)
+        if (
+            path.is_absolute()
+            or not path.parts
+            or ".." in path.parts
+            or "\\" in value
+            or path.parts[0] == "manifest.json"
+        ):
+            raise ValueError(
+                "artifact destination must be a relative path without '..' or "
+                "backslashes, outside the reserved manifest.json path"
+            )
+        return str(path)
+
 
 async def collect(
     runtime: Runtime,
@@ -136,6 +160,100 @@ async def collect(
     return collected
 
 
+def save_artifacts(
+    directory: Path,
+    collected: dict[str, bytes | None],
+    artifacts: list[Artifact],
+    *,
+    workdir: str = "/",
+) -> None:
+    """Save collected files at their declared destinations, with a manifest.
+
+    Unpack each validated archive in isolation before moving its files into the
+    saved tree. Failed or conflicting entries are recorded without losing the
+    completed trace. Source-mirrored directories may merge: collection splits
+    nested declarations out of the implicit convention sweep.
+    """
+    declared = {str(PurePosixPath(workdir) / a.source): a for a in artifacts}
+    manifest: list[dict] = []
+    claims: list[tuple[PurePosixPath, bool]] = []
+    directory.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(dir=directory.parent) as temporary:
+        saved = Path(temporary) / "artifacts"
+        saved.mkdir()
+        (saved / "manifest.json").touch()
+        for source, archive in collected.items():
+            artifact = declared.get(source, Artifact(source=source))
+            destination = artifact.destination or artifact.source.lstrip("/")
+            entry = {
+                "source": source,
+                "service": getattr(artifact, "service", "main"),
+                "destination": destination,
+                "status": "empty",
+            }
+            manifest.append(entry)
+            try:
+                _validate_restore(source, archive)
+                Artifact.validate_destination(artifact.destination)
+                target = PurePosixPath(destination)
+                entry["destination"] = str(target)
+                if archive is None:
+                    continue
+                with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar:
+                    members = tar.getmembers()
+                    if not members:
+                        continue
+                    root = PurePosixPath(source).relative_to("/")
+                    paths = [
+                        saved / target / PurePosixPath(member.name).relative_to(root)
+                        for member in members
+                    ]
+                    # Explicit renames claim a whole subtree. Source-mirrored
+                    # entries may share directories, but never overwrite files.
+                    if any(
+                        (artifact.destination or renamed)
+                        and (
+                            target.is_relative_to(prior) or prior.is_relative_to(target)
+                        )
+                        for prior, renamed in claims
+                    ) or any(
+                        path.is_symlink()
+                        or (path.exists() and not (member.isdir() and path.is_dir()))
+                        or any(
+                            parent.is_symlink()
+                            or (parent.exists() and not parent.is_dir())
+                            for parent in path.parents
+                            if parent.is_relative_to(saved)
+                        )
+                        for member, path in zip(members, paths, strict=True)
+                    ):
+                        entry["status"] = "skipped"
+                        entry["error"] = (
+                            "destination overlaps a previously saved artifact"
+                        )
+                        logger.warning("artifact %r: %s", source, entry["error"])
+                        continue
+                    with tempfile.TemporaryDirectory(dir=temporary) as unpacked:
+                        tar.extractall(unpacked, filter="data")
+                        for member, path in zip(members, paths, strict=True):
+                            if member.isdir():
+                                path.mkdir(parents=True, exist_ok=True)
+                            else:
+                                path.parent.mkdir(parents=True, exist_ok=True)
+                                (Path(unpacked) / member.name).replace(path)
+                    claims.append((target, artifact.destination is not None))
+                    entry["status"] = "ok"
+            except (OSError, ValueError, RuntimeError, tarfile.TarError) as exc:
+                entry.update(status="failed", error=str(exc))
+                logger.warning("failed to save artifact %r: %s", source, exc)
+        (saved / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+        if directory.is_symlink():
+            raise ValueError(f"artifact directory must not be a symlink: {directory}")
+        if directory.exists():
+            shutil.rmtree(directory)
+        saved.replace(directory)
+
+
 async def restore(runtime: Runtime, collected: dict[str, bytes | None]) -> None:
     """Extract `collected` in `runtime` at the original absolute paths.
 

{verifiers-0.4.1.dev34 → verifiers-0.4.1.dev36}/verifiers/v1/utils/trace_store.py

@@ -14,6 +14,7 @@ from verifiers.v1.state import StateT
 from verifiers.v1.task import DataT
 from verifiers.v1.trace import AgentConfigT, Trace
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.artifacts import save_artifacts
 
 TRACES_FILE = "traces.jsonl"
 type_adapter = cache(TypeAdapter)
@@ -25,6 +26,18 @@ def write_episode(
     """Serialize and append one rollout episode in the worker thread."""
     # Preserve fields declared by typed Trace subclasses nested in the episode.
     data = type_adapter(type(episode)).dump_json(episode, exclude_none=True)
+    for trace in episode.traces:
+        if trace.state.artifacts:
+            if trace.id == ".." or Path(trace.id).parts != (trace.id,):
+                raise ValueError(
+                    f"trace ID must be a single path component: {trace.id!r}"
+                )
+            save_artifacts(
+                results_dir / "artifacts" / trace.id,
+                trace.state.artifacts,
+                trace.task.data.artifacts,
+                workdir=getattr(trace.agent.runtime, "workdir", None) or "/",
+            )
     with (results_dir / TRACES_FILE).open("ab") as f:
         f.write(data + b"\n")