verifiers 0.2.2.dev49__tar.gz → 0.2.2.dev57__tar.gz

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev49
+Version: 0.2.2.dev57
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/pyproject.toml

@@ -234,6 +234,7 @@ markers = [
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "null: v1 e2e cases on the null harness",
     "bash: v1 e2e cases on the bash harness",
+    "browser_use: v1 e2e cases on the browser_use harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
     "pi: v1 e2e cases on the pi harness",

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/conftest.py

@@ -135,10 +135,6 @@ def _eval_config(
     `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
     harness one); `model` overrides the default text model (e.g. a VLM for an image task).
 
-    `temperature=0` (greedy) makes the run reproducible; `max_tokens` is generous headroom,
-    not a target — these trivial tasks finish in a few hundred tokens, so capping tighter only
-    risks truncating the reasoning before the answer (which tanks the reward).
-
     `harness=None` leaves every seat on its own story — the multi-agent case: there
     is no run-level harness, so a single-agent test's `harness` lands on the `agent`
     seat and a multi-agent test pins its seats through `env` role fields instead."""
@@ -175,7 +171,6 @@ def _eval_config(
         num_rollouts=n,
         sampling={
             "max_tokens": max_tokens,
-            "temperature": 0,
             "reasoning_effort": reasoning_effort,
         },
         rich=False,
@@ -219,8 +214,8 @@ def run_v1_server():
 
 @pytest.fixture
 async def live_ctx():
-    """A live `ModelContext` (the e2e default model + endpoint, greedy) for driving
-    `Agent` directly — the agent-surface counterpart of `run_v1`."""
+    """A live `ModelContext` (the e2e default model + endpoint, provider-default
+    sampling) for driving `Agent` directly — the agent-surface counterpart of `run_v1`."""
     from verifiers.v1.clients import EvalClientConfig, ModelContext, resolve_client
     from verifiers.v1.types import SamplingConfig
 
@@ -229,7 +224,7 @@ async def live_ctx():
         yield ModelContext(
             model="deepseek/deepseek-v4-flash",
             client=client,
-            sampling=SamplingConfig(max_tokens=2048, temperature=0),
+            sampling=SamplingConfig(max_tokens=2048),
         )
     finally:
         await client.close()
@@ -252,7 +247,7 @@ def run_v0():
             legacy={"id": env_id, "args": args or {}},
             num_tasks=1,
             num_rollouts=n,
-            sampling={"max_tokens": max_tokens, "temperature": 0},
+            sampling={"max_tokens": max_tokens},
             rich=False,
             output_dir=output_dir,
         )

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/fixtures/counter_tool_v1.py

@@ -24,7 +24,9 @@ class CounterTaskConfig(vf.TaskConfig):
 
 
 class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
-    tools = (CounterToolset,)
+    @classmethod
+    def toolsets(cls, config: CounterTaskConfig) -> list[vf.Toolset]:
+        return [CounterToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def counted(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/fixtures/echo_acp_resume_v1.py

@@ -28,7 +28,9 @@ class ACPResumeConfig(vf.TasksetConfig):
 
 
 class ACPResumeTask(vf.Task[vf.TaskData, vf.State, ACPResumeTaskConfig]):
-    tools = (ResumeToolset,)
+    @classmethod
+    def toolsets(cls, config: ACPResumeTaskConfig) -> list[vf.Toolset]:
+        return [ResumeToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def resumed(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/fixtures/echo_tool_v1.py

@@ -29,7 +29,9 @@ class EchoToolTaskConfig(vf.TaskConfig):
 
 
 class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
-    tools = (EchoToolset,)
+    @classmethod
+    def toolsets(cls, config: EchoToolTaskConfig) -> list[vf.Toolset]:
+        return [EchoToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/fixtures/tool_response_image_v1.py

@@ -26,8 +26,16 @@ class VisionToolset(vf.Toolset[vf.ToolsetConfig]):
         ]
 
 
-class ToolResponseImageTask(vf.Task):
-    tools = (VisionToolset,)
+class ToolResponseImageTaskConfig(vf.TaskConfig):
+    tools: vf.ToolsetConfig = vf.ToolsetConfig()
+
+
+class ToolResponseImageTask(
+    vf.Task[vf.TaskData, vf.State, ToolResponseImageTaskConfig]
+):
+    @classmethod
+    def toolsets(cls, config: ToolResponseImageTaskConfig) -> list[vf.Toolset]:
+        return [VisionToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def preserved_image_tool_result(self, trace: vf.Trace) -> float:
@@ -39,7 +47,13 @@ class ToolResponseImageTask(vf.Task):
         return 0.0
 
 
-class ToolResponseImageTaskset(vf.Taskset[ToolResponseImageTask, vf.TasksetConfig]):
+class ToolResponseImageConfig(vf.TasksetConfig):
+    task: ToolResponseImageTaskConfig = ToolResponseImageTaskConfig()
+
+
+class ToolResponseImageTaskset(
+    vf.Taskset[ToolResponseImageTask, ToolResponseImageConfig]
+):
     def load(self) -> list[ToolResponseImageTask]:
         return [
             ToolResponseImageTask(

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/test_color_codeword_tasks.py

@@ -10,9 +10,7 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
         return render(image)
 
     monkeypatch.setattr(taskset, "image_data_url", counted)
-    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
-        num_tasks=100
-    )
+    tasks = list(taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).head(100))
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_RGB.values())

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/test_e2e.py

@@ -109,7 +109,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
     # The seat's resolved identity rides the trace (policy metadata for trainers).
-    assert trace.agent is not None and trace.agent.config.sampling.temperature == 0
+    assert trace.agent is not None and trace.agent.config.sampling.max_tokens == 2048
     # Every sampled turn has one per-call record, linked to its assistant node.
     sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
     assert [c.node for c in trace.calls if c.error is None] == sampled
@@ -118,6 +118,26 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
         assert call.time.duration > 0
 
 
+@pytest.mark.e2e
+@pytest.mark.browser_use
+@pytest.mark.docker
+async def test_browser_use(run_v1, tmp_path):
+    """The browser_use harness runs in an explicitly browser-capable runtime."""
+    image = (
+        "mcr.microsoft.com/playwright/python:v1.61.0-noble@"
+        "sha256:a9731514f24121d1dcd25d58d0a38146646d290a5998fd80d3e533e7b5e21c69"
+    )
+    (trace,) = await run_v1(
+        "echo-v1",
+        harness="browser_use",
+        runtime={"type": "docker", "image": image},
+        output_dir=tmp_path,
+        max_turns=2,
+    )
+    assert trace.ok
+    assert trace.reward == 1.0
+
+
 @pytest.mark.e2e
 @pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True)
 async def test_user(run_v1, harness_runtime, tmp_path):
@@ -335,6 +355,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
         runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=10,
+        max_tokens=8192,
     )
     assert trace.ok
     assert trace.num_turns >= 1  # ran a command, then finished
@@ -366,7 +387,7 @@ async def test_multi_agent_env(run_v1, tmp_path):
     # agent info (completion order — the gathered seats land in either order).
     (line,) = (tmp_path / "traces.jsonl").read_text().splitlines()
     row = json.loads(line)
-    assert row["env"] == "duet-v1"
+    assert row["env"] == {"id": "duet-v1"}
     by_name = {t["agent"]["name"]: t for t in row["traces"]}
     assert set(by_name) == {"a", "b"}
     assert by_name["a"]["agent"]["trainable"] is True
@@ -401,21 +422,21 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     model's call. Exercises the config surface too: a policy-only prompt
     override (the verdict contract is appended regardless) and
     reward-composition weights."""
+    policy = tmp_path / "judge_policy.txt"
+    policy.write_text("Check EMPIRICALLY that the agent echoed the word back.")
     traces = await run_v1(
         "echo-v1",
-        harness=None,  # seats pin their own harness; there is no run-level one
+        harness=None,
         env={
             "id": "agentic-judge",
-            # The solver owns the shared box, so the container is pinned here.
             "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}},
-            # The judge reads the trace and reasons before it writes the
-            # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
                 "harness": {"id": "bash"},
                 "max_output_tokens": 8192,
             },
             "task": {
-                "prompt": "Check EMPIRICALLY that the agent echoed the word back.",
+                "prompt": {"path": str(policy)},
+                "hint": "Do not rely on README.md",
             },
             "score": {"task_weight": 0.5},
         },
@@ -431,7 +452,7 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     # The task's own reward keeps its raw score; the rescale lands on the weight.
     assert solver.rewards["echoed"].score == 1.0
     assert solver.rewards["echoed"].weight == 0.5
-    assert isinstance(judge.info.get("verdict"), dict)  # scraped off the box
+    assert isinstance(judge.info.get("verdict"), dict)
     assert 0.0 <= solver.rewards["judge"].score <= 1.0
 
 

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/tests/v1/test_judges.py

@@ -371,32 +371,20 @@ async def test_rubric_view_full_trace(tmp_path, fake_judge_model):
     assert all("SECRET REASONING" not in prompt for prompt in fake_judge_model)
 
 
-async def test_config_prompt_overrides_class_template(fake_judge_model):
-    judge = vf.ReferenceJudge(
-        vf.ReferenceJudgeConfig(prompt="Q:{question} A:{answer} R:{response}")
-    )
+async def test_config_prompt_overrides_class_template(tmp_path, fake_judge_model):
+    # Config `prompt` is a file path; the same {field} placeholders work.
+    file = tmp_path / "judge.txt"
+    file.write_text("Q:{question} A:{answer} R:{response}")
+    judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt=file))
     assert judge.build_messages(question="q", answer="a", response="r") == "Q:q A:a R:r"
     # A template needn't use every evaluate field: score also passes {positive}/{negative},
     # which str.format ignores when the (custom) prompt doesn't reference them.
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 1.0
     assert fake_judge_model[0] == "Q:Capital of France? A:Paris R:It is Paris."
-
-
-async def test_prompt_file(tmp_path, fake_judge_model):
-    # The prompt template can come from a file; the same {field} placeholders work.
-    file = tmp_path / "judge.txt"
-    file.write_text("Q:{question} A:{answer} R:{response}")
-    trace = make_trace()
-    judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt_file=file))
-    assert await judge.score(trace.task.data, trace) == 1.0
-    assert fake_judge_model[0] == "Q:Capital of France? A:Paris R:It is Paris."
     # a bad path fails at judge construction, not mid-eval at score time
     with pytest.raises(FileNotFoundError):
-        vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt_file=tmp_path / "missing.txt"))
-    # inline and file prompts are mutually exclusive
-    with pytest.raises(ValueError, match="not both"):
-        vf.ReferenceJudgeConfig(prompt="inline", prompt_file=file)
+        vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt=tmp_path / "missing.txt"))
 
 
 # --- reference input/verdict knobs ------------------------------------------------------------

verifiers-0.2.2.dev57/tests/v1/test_taskset.py

@@ -0,0 +1,93 @@
+"""`Taskset` iteration and the `head`/`shuffle` views over list, generator,
+and `INFINITE` `load` implementations."""
+
+import itertools
+
+import pytest
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        for i in range(10):
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+def idxs(tasks) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def test_head_bounds_an_infinite_taskset() -> None:
+    tasks = list(InfiniteTaskset(vf.TasksetConfig()).head(5))
+    assert idxs(tasks) == [0, 1, 2, 3, 4]
+
+
+def test_iteration_materializes_a_finite_taskset() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    assert len(list(taskset)) == 10
+    assert idxs(taskset.head(4)) == [0, 1, 2, 3]
+
+
+def test_head_returns_the_same_taskset_type() -> None:
+    view = FiniteTaskset(vf.TasksetConfig()).head(4)
+    assert isinstance(view, FiniteTaskset)
+    assert view.task_type() is CountTask
+
+
+def test_shuffle_samples_the_whole_taskset_reproducibly() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    first = idxs(taskset.shuffle().head(5))
+    assert first == idxs(taskset.shuffle().head(5))
+    assert len(first) == 5 and set(first) <= set(range(10))
+    assert first != [0, 1, 2, 3, 4]  # sampled from the whole set, not the head
+
+
+def test_shuffle_seed_changes_the_sample() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    seeded = idxs(taskset.shuffle(seed=7).head(5))
+    assert seeded == idxs(taskset.shuffle(seed=7).head(5))  # reproducible per seed
+    assert seeded != idxs(taskset.shuffle().head(5))  # differs from the default seed
+
+
+def test_shuffle_raises_on_infinite() -> None:
+    with pytest.raises(ValueError, match="infinite"):
+        InfiniteTaskset(vf.TasksetConfig()).shuffle()
+
+
+def test_head_then_shuffle_bounds_an_infinite_taskset() -> None:
+    head = idxs(InfiniteTaskset(vf.TasksetConfig()).head(5).shuffle())
+    assert sorted(head) == [0, 1, 2, 3, 4]
+    assert head != [0, 1, 2, 3, 4]  # shuffled within the bounded head
+
+
+def test_head_only_builds_what_the_run_takes() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+    list(RecordingTaskset(vf.TasksetConfig()).head(3))
+    assert built == [0, 1, 2]
+
+
+def test_views_do_not_mutate_the_base_taskset() -> None:
+    taskset = InfiniteTaskset(vf.TasksetConfig())
+    view = taskset.head(3)
+    assert view.INFINITE is False and taskset.INFINITE is True
+    assert idxs(taskset.head(2)) == [0, 1]  # base iterates untransformed

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/verifiers/v1/__init__.py

@@ -4,6 +4,12 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.acp import ACP
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
+from verifiers.v1.artifacts import (
+    ARTIFACTS_DIR,
+    Artifact,
+    collect,
+    restore,
+)
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -109,10 +115,10 @@ from verifiers.v1.taskset import Taskset
 from verifiers.v1.trace import (
     TRACE_VERSION,
     AgentInfo,
+    AgentSpan,
     Branch,
     Error,
     EvalRunInfo,
-    GenerationSpan,
     ModelCall,
     Reward,
     RunInfo,
@@ -138,7 +144,6 @@ from verifiers.v1.types import (
     Response,
     Sampling,
     SamplingConfig,
-    StrictBaseModel,
     SystemMessage,
     TextContentPart,
     Tool,
@@ -171,7 +176,6 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Response",
     "Sampling",
     "SamplingConfig",
-    "StrictBaseModel",
     "SystemMessage",
     "TextContentPart",
     "Tool",
@@ -207,7 +211,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Timing",
     "TimeSpan",
     "TimeSplit",
-    "GenerationSpan",
+    "AgentSpan",
     "Error",
     # decorators
     "stop",
@@ -297,6 +301,11 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "PATCH_CAP_BYTES",
     "capture_patch",
     "resolve_head",
+    # grading artifacts
+    "ARTIFACTS_DIR",
+    "Artifact",
+    "collect",
+    "restore",
     # scoring
     "compare_stdout_results",
     "extract_boxed_answer",

{verifiers-0.2.2.dev49 → verifiers-0.2.2.dev57}/verifiers/v1/agent.py

@@ -335,7 +335,7 @@ class Agent:
         if self._server is None:
             return None
         if self._server.tunnel is not None or (
-            run_is_local and not shared_tools and not type(task).tools
+            run_is_local and not shared_tools and not task.toolsets(task.config)
         ):
             return self._server
         return None
@@ -517,7 +517,10 @@ class Agent:
             )
             run_is_local = runtime_is_local(runtime_config)
         validate_pairing(
-            self.harness, type(task), runtime_config, shared_tools=shared_tools
+            self.harness,
+            type(task),
+            runtime_config,
+            tools=[*task.toolsets(task.config), *shared_tools.values()],
         )
         # Timeout precedence: agent-level wins, else the task's, else no limit.
         agent_timeout = (

verifiers-0.2.2.dev57/verifiers/v1/artifacts.py

@@ -0,0 +1,142 @@
+"""Artifact collection and restoration across runtimes."""
+
+from __future__ import annotations
+
+import logging
+import shlex
+import uuid
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes import Runtime
+
+logger = logging.getLogger(__name__)
+
+ARTIFACTS_DIR = "/logs/artifacts"
+"""Implicit artifact directory; tasks that write here need no declaration."""
+
+MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
+"""Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
+agent's image, so the repo is already there and only its output has to travel."""
+
+
+class Artifact(BaseModel):
+    """One path to restore at the same location in another runtime."""
+
+    source: str
+    exclude: list[str] = Field(default_factory=list)
+    """`tar --exclude` patterns, applied when `source` is a directory."""
+
+
+async def collect(
+    runtime: Runtime, artifacts: list[Artifact] | None = None
+) -> dict[str, bytes]:
+    """Tar the convention dir and every declared path out of `runtime`.
+
+    Keyed by source path; the values are tar archives. Insertion order is the order
+    they were declared, and a path cannot be collected twice.
+
+    A declared source that is missing raises: it was declared because grading needs it,
+    and grading a partial state scores the rollout wrong rather than failing it. The
+    implicit convention sweep is exempt — most tasks never write there.
+
+    Each source is archived separately so its exclude patterns stay local.
+    """
+    # Resolve relative sources against the runtime workdir. Joining also normalises
+    # `/work/` to `/work`, so one tree cannot key two entries (the source is both the
+    # dict key and `restore`'s rm -rf target).
+    workdir = PurePosixPath(getattr(runtime.config, "workdir", "") or "/")
+    declared = [
+        a.model_copy(update={"source": str(workdir / a.source)})
+        for a in artifacts or []
+    ]
+    convention = PurePosixPath(ARTIFACTS_DIR)
+    sweep = not any(
+        (p := PurePosixPath(a.source)) == convention
+        or p.is_relative_to(convention)
+        or convention.is_relative_to(p)
+        for a in declared
+    )
+    entries = ([Artifact(source=ARTIFACTS_DIR)] if sweep else []) + declared
+
+    collected: dict[str, bytes] = {}
+    budget = MAX_ARTIFACT_BYTES
+    for artifact in entries:
+        source = artifact.source
+        if (await runtime.run(["test", "-e", source], {})).exit_code != 0:
+            if sweep and source == ARTIFACTS_DIR:
+                continue
+            raise RuntimeError(
+                f"declared artifact {source!r} does not exist in the runtime"
+            )
+        archive = await _tar_out(runtime, artifact, budget)
+        budget -= len(archive)
+        collected[source] = archive
+
+    logger.debug("collected artifact roots: %s", list(collected))
+    return collected
+
+
+async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None:
+    """Extract `collected` in `runtime` at the original absolute paths."""
+    if not collected:
+        return
+    # Restoring into the subprocess runtime would extract absolute paths onto the
+    # developer's filesystem, so refuse it before any archive reaches the host.
+    if getattr(runtime.config, "type", None) == "subprocess":
+        raise RuntimeError(
+            "refusing to restore artifacts into the subprocess runtime: extraction "
+            "writes to absolute paths on the host. Grade in a container."
+        )
+    # Clear every root up front, not per entry: a later nested root would otherwise
+    # delete content an earlier one just restored. Clearing also drops any file or
+    # symlink the image left at the target.
+    roots = " ".join(shlex.quote(root) for root in collected)
+    await _run(runtime, f"rm -rf -- {roots}", "clear artifact roots")
+    for root, archive in collected.items():
+        path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar"
+        await runtime.write(path, archive)
+        await _run(
+            runtime,
+            f"tar -xf {shlex.quote(path)} -C / && rm -f {shlex.quote(path)}",
+            f"restore artifact {root!r}",
+        )
+
+
+async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes:
+    path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar"
+    excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude)
+    try:
+        await _run(
+            runtime,
+            f"tar -cf {shlex.quote(path)} -C / {excludes} -- "
+            f"{shlex.quote(artifact.source.lstrip('/'))}",
+            f"collect artifact {artifact.source!r}",
+        )
+        # Size it in the box: an oversized collection is refused before it reaches host
+        # memory, not after.
+        sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {})
+        if (raw := sized.stdout.strip()).isdigit() and int(raw) > budget:
+            raise RuntimeError(
+                f"artifact {artifact.source!r} takes the collection over the "
+                f"{MAX_ARTIFACT_BYTES} byte limit. The grading box boots from the "
+                "agent's image, so only the delta needs to travel — narrow the source "
+                "or add `exclude` patterns."
+            )
+        return await runtime.read(path)
+    finally:
+        # Best-effort: the box is about to be destroyed and the name is unique per call.
+        try:
+            await runtime.run(["rm", "-f", path], {})
+        except Exception:
+            logger.debug("failed to remove %s", path, exc_info=True)
+
+
+async def _run(runtime: Runtime, command: str, action: str) -> None:
+    result = await runtime.run(["sh", "-c", command], {})
+    if result.exit_code:
+        detail = (result.stderr or result.stdout).strip()[-500:]
+        raise RuntimeError(f"failed to {action}: {detail}")