verifiers 0.2.2.dev52__tar.gz → 0.2.2.dev53__tar.gz

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev52
+Version: 0.2.2.dev53
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/tests/v1/test_e2e.py

@@ -335,6 +335,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
         runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=10,
+        max_tokens=8192,
     )
     assert trace.ok
     assert trace.num_turns >= 1  # ran a command, then finished
@@ -405,18 +406,18 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     policy.write_text("Check EMPIRICALLY that the agent echoed the word back.")
     traces = await run_v1(
         "echo-v1",
-        harness=None,  # seats pin their own harness; there is no run-level one
+        harness=None,
         env={
             "id": "agentic-judge",
-            # The solver owns the shared box, so the container is pinned here.
             "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}},
-            # The judge reads the trace and reasons before it writes the
-            # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
                 "harness": {"id": "bash"},
                 "max_output_tokens": 8192,
             },
-            "task": {"prompt": str(policy)},
+            "task": {
+                "prompt": {"path": str(policy)},
+                "hint": "Do not rely on README.md",
+            },
             "score": {"task_weight": 0.5},
         },
         output_dir=tmp_path,
@@ -431,7 +432,7 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     # The task's own reward keeps its raw score; the rescale lands on the weight.
     assert solver.rewards["echoed"].score == 1.0
     assert solver.rewards["echoed"].weight == 0.5
-    assert isinstance(judge.info.get("verdict"), dict)  # scraped off the box
+    assert isinstance(judge.info.get("verdict"), dict)
     assert 0.0 <= solver.rewards["judge"].score <= 1.0
 
 

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/verifiers/v1/__init__.py

@@ -4,6 +4,12 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.acp import ACP
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
+from verifiers.v1.artifacts import (
+    ARTIFACTS_DIR,
+    Artifact,
+    collect,
+    restore,
+)
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -297,6 +303,11 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "PATCH_CAP_BYTES",
     "capture_patch",
     "resolve_head",
+    # grading artifacts
+    "ARTIFACTS_DIR",
+    "Artifact",
+    "collect",
+    "restore",
     # scoring
     "compare_stdout_results",
     "extract_boxed_answer",

verifiers-0.2.2.dev53/verifiers/v1/artifacts.py

@@ -0,0 +1,144 @@
+"""Artifact collection and restoration across runtimes."""
+
+from __future__ import annotations
+
+import logging
+import shlex
+import uuid
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING
+
+from pydantic import Field
+
+from verifiers.v1.types import StrictBaseModel
+
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes import Runtime
+
+logger = logging.getLogger(__name__)
+
+ARTIFACTS_DIR = "/logs/artifacts"
+"""Implicit artifact directory; tasks that write here need no declaration."""
+
+MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
+"""Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
+agent's image, so the repo is already there and only its output has to travel."""
+
+
+class Artifact(StrictBaseModel):
+    """One path to restore at the same location in another runtime."""
+
+    source: str
+    exclude: list[str] = Field(default_factory=list)
+    """`tar --exclude` patterns, applied when `source` is a directory."""
+
+
+async def collect(
+    runtime: Runtime, artifacts: list[Artifact] | None = None
+) -> dict[str, bytes]:
+    """Tar the convention dir and every declared path out of `runtime`.
+
+    Keyed by source path; the values are tar archives. Insertion order is the order
+    they were declared, and a path cannot be collected twice.
+
+    A declared source that is missing raises: it was declared because grading needs it,
+    and grading a partial state scores the rollout wrong rather than failing it. The
+    implicit convention sweep is exempt — most tasks never write there.
+
+    Each source is archived separately so its exclude patterns stay local.
+    """
+    # Resolve relative sources against the runtime workdir. Joining also normalises
+    # `/work/` to `/work`, so one tree cannot key two entries (the source is both the
+    # dict key and `restore`'s rm -rf target).
+    workdir = PurePosixPath(getattr(runtime.config, "workdir", "") or "/")
+    declared = [
+        a.model_copy(update={"source": str(workdir / a.source)})
+        for a in artifacts or []
+    ]
+    convention = PurePosixPath(ARTIFACTS_DIR)
+    sweep = not any(
+        (p := PurePosixPath(a.source)) == convention
+        or p.is_relative_to(convention)
+        or convention.is_relative_to(p)
+        for a in declared
+    )
+    entries = ([Artifact(source=ARTIFACTS_DIR)] if sweep else []) + declared
+
+    collected: dict[str, bytes] = {}
+    budget = MAX_ARTIFACT_BYTES
+    for artifact in entries:
+        source = artifact.source
+        if (await runtime.run(["test", "-e", source], {})).exit_code != 0:
+            if sweep and source == ARTIFACTS_DIR:
+                continue
+            raise RuntimeError(
+                f"declared artifact {source!r} does not exist in the runtime"
+            )
+        archive = await _tar_out(runtime, artifact, budget)
+        budget -= len(archive)
+        collected[source] = archive
+
+    logger.debug("collected artifact roots: %s", list(collected))
+    return collected
+
+
+async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None:
+    """Extract `collected` in `runtime` at the original absolute paths."""
+    if not collected:
+        return
+    # Restoring into the subprocess runtime would extract absolute paths onto the
+    # developer's filesystem, so refuse it before any archive reaches the host.
+    if getattr(runtime.config, "type", None) == "subprocess":
+        raise RuntimeError(
+            "refusing to restore artifacts into the subprocess runtime: extraction "
+            "writes to absolute paths on the host. Grade in a container."
+        )
+    # Clear every root up front, not per entry: a later nested root would otherwise
+    # delete content an earlier one just restored. Clearing also drops any file or
+    # symlink the image left at the target.
+    roots = " ".join(shlex.quote(root) for root in collected)
+    await _run(runtime, f"rm -rf -- {roots}", "clear artifact roots")
+    for root, archive in collected.items():
+        path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar"
+        await runtime.write(path, archive)
+        await _run(
+            runtime,
+            f"tar -xf {shlex.quote(path)} -C / && rm -f {shlex.quote(path)}",
+            f"restore artifact {root!r}",
+        )
+
+
+async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes:
+    path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar"
+    excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude)
+    try:
+        await _run(
+            runtime,
+            f"tar -cf {shlex.quote(path)} -C / {excludes} -- "
+            f"{shlex.quote(artifact.source.lstrip('/'))}",
+            f"collect artifact {artifact.source!r}",
+        )
+        # Size it in the box: an oversized collection is refused before it reaches host
+        # memory, not after.
+        sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {})
+        if (raw := sized.stdout.strip()).isdigit() and int(raw) > budget:
+            raise RuntimeError(
+                f"artifact {artifact.source!r} takes the collection over the "
+                f"{MAX_ARTIFACT_BYTES} byte limit. The grading box boots from the "
+                "agent's image, so only the delta needs to travel — narrow the source "
+                "or add `exclude` patterns."
+            )
+        return await runtime.read(path)
+    finally:
+        # Best-effort: the box is about to be destroyed and the name is unique per call.
+        try:
+            await runtime.run(["rm", "-f", path], {})
+        except Exception:
+            logger.debug("failed to remove %s", path, exc_info=True)
+
+
+async def _run(runtime: Runtime, command: str, action: str) -> None:
+    result = await runtime.run(["sh", "-c", command], {})
+    if result.exit_code:
+        detail = (result.stderr or result.stdout).strip()[-500:]
+        raise RuntimeError(f"failed to {action}: {detail}")

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/verifiers/v1/envs/agentic_judge/__init__.py

@@ -4,6 +4,7 @@ from verifiers.v1.envs.agentic_judge.env import (
     Criterion,
     JudgeTaskConfig,
     ScoreConfig,
+    TextFile,
 )
 
 __all__ = [
@@ -12,4 +13,5 @@ __all__ = [
     "Criterion",
     "JudgeTaskConfig",
     "ScoreConfig",
+    "TextFile",
 ]

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/verifiers/v1/envs/agentic_judge/env.py

@@ -1,14 +1,16 @@
-"""agentic-judge: a solver plays the task, a judge verifies it in the same box.
+"""agentic-judge: a solver plays the task, then a judge verifies the work.
 
-A reusable env (`--env.id agentic-judge` over any taskset): the box is
-provisioned from the solver's runtime policy, the solver plays the task in it,
-and a code-executing judge then inspects the work as the agent left it, with
-the solver's full trace record uploaded at `/tmp/trace.json`. The judge grades
+A reusable env (`--env.id agentic-judge` over any taskset). The solver plays the
+task in a container provisioned from its runtime policy; the judge then grades
 rubric criteria (`[env.task]`: policy prompt, criteria file) and writes its
-verdicts to `/tmp/verdict.json`; `finalize()` validates them strictly onto the
-solver's trace — `judge/<name>` metrics plus a weighted-mean `judge` reward,
-composed with the taskset's own rewards via `[env.score]` (judge-only by
-default).
+verdicts to `/tmp/verdict.json`, with the solver's full trace record uploaded at
+`/tmp/trace.json`. `finalize()` validates them strictly onto the solver's trace —
+`judge/<name>` metrics plus a weighted-mean `judge` reward, composed with the
+taskset's own rewards via `[env.score]` (judge-only by default).
+
+`--env.share-runtime` controls whether the judge uses the solver's runtime. It is
+enabled by default. When disabled, the judge gets a fresh runtime containing the
+task's collected artifacts.
 """
 
 import json
@@ -21,6 +23,7 @@ from pydantic import Field, field_validator
 
 import verifiers.v1 as vf
 from verifiers.v1.types import StrictBaseModel
+from verifiers.v1.utils.compile import validate_pairing
 
 VERDICT_FILE = "/tmp/verdict.json"
 TRACE_FILE = "/tmp/trace.json"
@@ -97,19 +100,31 @@ def _render(template: str, **fields: str) -> str:
     return pattern.sub(lambda m: fields[m.group(1)], template)
 
 
-SANDBOX_NOTE = f"""\
+_RECORD_NOTE = f"""\
+The agent's raw trace record (JSON: messages, tool calls, and its `info`
+artifacts) is written by the harness — not the agent — at `{TRACE_FILE}`. The
+record can be very large — never dump it whole; peek selectively (list its
+keys, then slice out specific fields with python or jq) and pull only what you
+need. It is complete — it may also carry the task's own scores/metrics and
+reference material (a gold answer, a reference solution, held-out tests). Those
+are context, not your standard: recorded scores can be wrong and references can
+be narrower than the task; do not over-index on how a reference solves it. Your
+verdict is what YOU verified by execution."""
+
+SHARED_WORKSPACE_NOTE = f"""\
+## Your workspace
+
+The graded agent worked in this sandbox. Its edits and any scoring side effects
+are present. {_RECORD_NOTE}"""
+
+ISOLATED_WORKSPACE_NOTE = f"""\
 ## Your workspace
 
-Your sandbox is the SAME box the graded agent worked in, in the state the agent
-left it — its edits (and any scoring side effects) are applied. The agent's raw
-trace record (JSON: messages, tool calls, and its `info` artifacts) is uploaded
-at `{TRACE_FILE}`. The record can be very large — never dump it whole; peek
-selectively (list its keys, then slice out specific fields with python or jq)
-and pull only what you need. It is complete — it may also carry the task's own
-scores/metrics and reference material (a gold answer, a reference solution,
-held-out tests). Those are context, not your standard: recorded scores can be
-wrong and references can be narrower than the task; do not over-index on how a
-reference solves it. Your verdict is what YOU verified by execution."""
+This is a fresh sandbox built from the task's image. The task's published
+artifacts were restored at their original paths; other changes made by the
+graded agent are not present. The usual artifact location is
+`{vf.ARTIFACTS_DIR}/` (for code tasks, typically a patch to read or apply), plus
+any task-declared paths. {_RECORD_NOTE}"""
 
 HINT_SECTION = """\
 ## Hints
@@ -125,13 +140,30 @@ class JudgeTask(vf.Task):
 
     NEEDS_CONTAINER = True
 
-    def __init__(self, data: vf.TaskData, files: dict[str, bytes]) -> None:
+    def __init__(
+        self,
+        data: vf.TaskData,
+        files: dict[str, bytes],
+        artifacts: dict[str, bytes],
+    ) -> None:
         super().__init__(data)
         self.files = files
+        self.artifacts = artifacts
 
     @classmethod
-    def from_trace(cls, solution: vf.Trace, config: "JudgeTaskConfig") -> "JudgeTask":
-        """Mint the judge's task from the solver's finished trace."""
+    def from_trace(
+        cls,
+        solution: vf.Trace,
+        config: "JudgeTaskConfig",
+        share_runtime: bool = True,
+    ) -> "JudgeTask":
+        """Mint the judge's task from the solver's finished trace.
+
+        `share_runtime` selects both the workspace note and artifact transport. In
+        the solver's box the published artifacts are already on disk, so none
+        travel; a fresh box gets the collected set, restored by `setup` at the
+        paths they had.
+        """
         solved = solution.task.data
         files = {TRACE_FILE: json.dumps(solution.to_record()).encode()}
         template = config.build_prompt()
@@ -139,7 +171,10 @@ class JudgeTask(vf.Task):
         if "{prompt}" not in template:
             # A policy that doesn't place the task statement itself still needs it.
             body += "\n\n" + _render(TASK_SECTION, prompt=solved.prompt_text)
-        sections = [body, _verdict_section(config.criteria()), SANDBOX_NOTE]
+        workspace_note = (
+            SHARED_WORKSPACE_NOTE if share_runtime else ISOLATED_WORKSPACE_NOTE
+        )
+        sections = [body, _verdict_section(config.criteria()), workspace_note]
         if (hint := config.build_hint()) is not None:
             sections.insert(1, _render(HINT_SECTION, hint=hint))
         prompt = "\n\n".join(sections)
@@ -152,9 +187,11 @@ class JudgeTask(vf.Task):
                 resources=solved.resources,
             ),
             files=files,
+            artifacts={} if share_runtime else solution.state.artifacts,
         )
 
     async def setup(self, trace: vf.Trace, runtime: vf.Runtime) -> None:
+        await vf.restore(runtime, self.artifacts)
         # The solver had this box first: a pre-seeded verdict must never read as
         # the judge's own, and a file (or planted symlink) at an upload path must
         # never survive it — a symlinked TRACE_FILE would redirect the write onto
@@ -177,34 +214,53 @@ class JudgeTask(vf.Task):
         trace.info["verdict"] = json.loads(raw)
 
 
+class TextFile(vf.BaseConfig):
+    """An explicit file-backed text value for config formats without `Path` values."""
+
+    path: Path
+
+
+TextSource = str | Path | TextFile
+
+
 class JudgeTaskConfig(vf.BaseConfig):
     """The judge's minted task: the grading policy and what lands in its box."""
 
-    prompt: Path | None = None
-    """Grading-policy file. Replaces only the policy body — the verdict contract
-    and workspace note are always appended. May reference `{prompt}` (the solver
-    task's prompt); if it doesn't, the task statement is appended after."""
-    hint: Path | None = None
-    """Optional hints file injected as their own section: task-family pointers
-    into the trace or box — e.g. for math, where the reference answer lives in
-    the record; for SWE, to diff the repo or read `info.patch`."""
+    prompt: TextSource | None = None
+    """Grading policy: a string is inline text; a `Path` in Python or
+    `{ path = "policy.md" }` in config reads a file. Replaces only the policy
+    body — the verdict contract and workspace note are always appended. May
+    reference `{prompt}` (the solver task's prompt); if it doesn't, the task
+    statement is appended after."""
+    hint: TextSource | None = None
+    """Optional hints, with the same explicit inline/file forms as `prompt`,
+    injected as their own section: task-family pointers into the trace or box —
+    e.g. for math, where the reference answer lives in the record; for SWE, to
+    diff the repo or read `info.patch`."""
     rubric: Path | None = None
     """Criteria the judge grades against: a `.toml`/`.json` file with a
     `criteria` list — the plugged rubric judge's format, so the same rubric
     files work for both. None grades the single built-in `solved` criterion."""
 
+    @staticmethod
+    def _resolve(value: TextSource) -> str:
+        if isinstance(value, str):
+            return value
+        path = value if isinstance(value, Path) else value.path
+        return path.read_text(encoding="utf-8")
+
     def build_prompt(self) -> str:
         if self.prompt is None:
             return GRADE_PROMPT + "\n\n" + TASK_SECTION
-        return self.prompt.read_text()
+        return self._resolve(self.prompt)
 
     def build_hint(self) -> str | None:
-        return self.hint.read_text() if self.hint is not None else None
+        return self._resolve(self.hint) if self.hint is not None else None
 
     def criteria(self) -> list[Criterion]:
         if self.rubric is None:
             return [SOLVED]
-        text = self.rubric.read_text()
+        text = self.rubric.read_text(encoding="utf-8")
         data = (
             tomllib.loads(text)
             if self.rubric.suffix.lower() == ".toml"
@@ -241,23 +297,23 @@ class ScoreConfig(vf.BaseConfig):
 
 class AgenticJudgeEnvConfig(vf.EnvConfig):
     solver: vf.AgentConfig = vf.AgentConfig()
-    """The solver agent. It owns the shared box, so its runtime must be a
-    container: `--env.solver.runtime.type docker|prime`."""
+    """The solver agent. Its runtime must be a container:
+    `--env.solver.runtime.type docker|prime`."""
     judge: vf.AgentConfig = vf.AgentConfig()
-    """The judge agent. It plays in the solver's box; its own runtime policy is
-    ignored (overwritten with the solver's)."""
+    """The judge agent. Its runtime is ignored when `share_runtime` is enabled;
+    otherwise it must be a container."""
+    share_runtime: bool = True
+    """Whether the judge grades in the solver's runtime."""
     task: JudgeTaskConfig = JudgeTaskConfig()
     score: ScoreConfig = ScoreConfig()
 
 
 class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
     def __init__(self, config: AgenticJudgeEnvConfig) -> None:
-        # The judge plays in the solver's box, so its effective runtime IS the
-        # solver's policy — aligning the config keeps the base env's subprocess
-        # warning and the runtime stamped on the judge's trace truthful.
-        config.judge = config.judge.model_copy(
-            update={"runtime": config.solver.runtime}
-        )
+        if config.share_runtime:
+            config.judge = config.judge.model_copy(
+                update={"runtime": config.solver.runtime}
+            )
         super().__init__(config)
         self._check_agents()
         # A missing policy file or a malformed rubric fails here, not mid-episode.
@@ -271,30 +327,42 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         judge = self._harnesses["judge"]
         if not judge.EXECUTES_CODE:
             raise ValueError(
-                "agentic-judge plays a code-executing judge in its own sandbox, but "
+                "agentic-judge requires a judge harness that can execute code, but "
                 f"harness {judge.config.id!r} is a tool-less chat loop — a verdict "
                 "that needs no execution is a plugged judge "
                 "(--env.taskset.task.judges), not an agent."
             )
         if isinstance(self.config.solver.runtime, vf.SubprocessConfig):
             raise TypeError(
-                "agentic-judge plays its judge in the solver's box, but the solver "
-                "(which provisions it) resolves to the subprocess runtime; use "
+                "agentic-judge requires the solver to run in a container, but it "
+                "resolves to the subprocess runtime; use "
                 "--env.solver.runtime.type docker or prime"
             )
+        validate_pairing(judge, JudgeTask, self.config.judge.runtime)
 
     async def setup(self, agents: vf.Agents) -> None:
         # The judge grades the policy; its tokens are never training data.
         agents.judge.trainable = False
 
     async def run(self, task: vf.Task, agents: vf.Agents) -> None:
-        async with agents.solver.provision(task) as box:
-            solution = await agents.solver.run(task, runtime=box)
-            judge_task = JudgeTask.from_trace(solution, self.config.task)
-            await agents.judge.run(judge_task, runtime=box)
+        if self.config.share_runtime:
+            async with agents.solver.provision(task) as box:
+                solution = await agents.solver.run(task, runtime=box)
+                judge_task = JudgeTask.from_trace(solution, self.config.task)
+                await agents.judge.run(judge_task, runtime=box)
+            return
+
+        solution = await agents.solver.run(task)
+        if not solution.ok:
+            return
+        await agents.judge.run(
+            JudgeTask.from_trace(solution, self.config.task, share_runtime=False)
+        )
 
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
         by_agent = {t.agent.name: t for t in episode.traces}
+        if "judge" not in by_agent:
+            return
         solution, verdict = by_agent["solver"], by_agent["judge"]
         data = verdict.info.get("verdict")
         if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/verifiers/v1/state.py

@@ -1,10 +1,10 @@
 """Mutable state shared within one rollout.
 
 Tool servers synchronize it through the interception state channel. It is excluded
-from serialized traces; persist artifacts in `Trace.info` instead.
+from serialized traces.
 """
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 from typing_extensions import TypeVar
 
 from verifiers.v1.types import StrictBaseModel
@@ -13,6 +13,7 @@ from verifiers.v1.utils.generic import concrete_type
 
 class State(StrictBaseModel):
     model_config = ConfigDict(ser_json_inf_nan="constants")
+    artifacts: dict[str, bytes] = Field(default_factory=dict)
 
 
 StateT = TypeVar("StateT", bound=State, default=State)

{verifiers-0.2.2.dev52 → verifiers-0.2.2.dev53}/verifiers/v1/task.py

@@ -34,6 +34,7 @@ from pydantic import ConfigDict, Field
 from pydantic_config import BaseConfig
 from typing_extensions import TypeVar
 
+from verifiers.v1.artifacts import Artifact
 from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.decorators import discover_decorated, invoke_all
 from verifiers.v1.errors import TaskError, boundary
@@ -102,6 +103,11 @@ class TaskData(StrictBaseModel):
     """Execution-time destinations denied by this task and combined with runtime
     blocks. Non-empty concrete allowlists cannot be combined with blocklists. Docker
     framework routes take precedence; ordinary Prime deny rules pass through unchanged."""
+    artifacts: list[Artifact] = Field(default_factory=list)
+    """Paths collected from one runtime and restored at the same locations in another,
+    on top of the implicitly collected `/logs/artifacts/` convention dir. Declare
+    runtime outputs that must cross that boundary. A declared path that is missing at
+    collection time fails the rollout."""
     timeout: TaskTimeout = TaskTimeout()
     resources: TaskResources = TaskResources()