verifiers 0.2.2.dev52__py3-none-any.whl → 0.2.2.dev54__py3-none-any.whl

verifiers/v1/__init__.py

@@ -4,6 +4,12 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.acp import ACP
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
+from verifiers.v1.artifacts import (
+    ARTIFACTS_DIR,
+    Artifact,
+    collect,
+    restore,
+)
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -109,10 +115,10 @@ from verifiers.v1.taskset import Taskset
 from verifiers.v1.trace import (
     TRACE_VERSION,
     AgentInfo,
+    AgentSpan,
     Branch,
     Error,
     EvalRunInfo,
-    GenerationSpan,
     ModelCall,
     Reward,
     RunInfo,
@@ -138,7 +144,6 @@ from verifiers.v1.types import (
     Response,
     Sampling,
     SamplingConfig,
-    StrictBaseModel,
     SystemMessage,
     TextContentPart,
     Tool,
@@ -171,7 +176,6 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Response",
     "Sampling",
     "SamplingConfig",
-    "StrictBaseModel",
     "SystemMessage",
     "TextContentPart",
     "Tool",
@@ -207,7 +211,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Timing",
     "TimeSpan",
     "TimeSplit",
-    "GenerationSpan",
+    "AgentSpan",
     "Error",
     # decorators
     "stop",
@@ -297,6 +301,11 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "PATCH_CAP_BYTES",
     "capture_patch",
     "resolve_head",
+    # grading artifacts
+    "ARTIFACTS_DIR",
+    "Artifact",
+    "collect",
+    "restore",
     # scoring
     "compare_stdout_results",
     "extract_boxed_answer",

verifiers/v1/artifacts.py

@@ -0,0 +1,142 @@
+"""Artifact collection and restoration across runtimes."""
+
+from __future__ import annotations
+
+import logging
+import shlex
+import uuid
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes import Runtime
+
+logger = logging.getLogger(__name__)
+
+ARTIFACTS_DIR = "/logs/artifacts"
+"""Implicit artifact directory; tasks that write here need no declaration."""
+
+MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
+"""Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
+agent's image, so the repo is already there and only its output has to travel."""
+
+
+class Artifact(BaseModel):
+    """One path to restore at the same location in another runtime."""
+
+    source: str
+    exclude: list[str] = Field(default_factory=list)
+    """`tar --exclude` patterns, applied when `source` is a directory."""
+
+
+async def collect(
+    runtime: Runtime, artifacts: list[Artifact] | None = None
+) -> dict[str, bytes]:
+    """Tar the convention dir and every declared path out of `runtime`.
+
+    Keyed by source path; the values are tar archives. Insertion order is the order
+    they were declared, and a path cannot be collected twice.
+
+    A declared source that is missing raises: it was declared because grading needs it,
+    and grading a partial state scores the rollout wrong rather than failing it. The
+    implicit convention sweep is exempt — most tasks never write there.
+
+    Each source is archived separately so its exclude patterns stay local.
+    """
+    # Resolve relative sources against the runtime workdir. Joining also normalises
+    # `/work/` to `/work`, so one tree cannot key two entries (the source is both the
+    # dict key and `restore`'s rm -rf target).
+    workdir = PurePosixPath(getattr(runtime.config, "workdir", "") or "/")
+    declared = [
+        a.model_copy(update={"source": str(workdir / a.source)})
+        for a in artifacts or []
+    ]
+    convention = PurePosixPath(ARTIFACTS_DIR)
+    sweep = not any(
+        (p := PurePosixPath(a.source)) == convention
+        or p.is_relative_to(convention)
+        or convention.is_relative_to(p)
+        for a in declared
+    )
+    entries = ([Artifact(source=ARTIFACTS_DIR)] if sweep else []) + declared
+
+    collected: dict[str, bytes] = {}
+    budget = MAX_ARTIFACT_BYTES
+    for artifact in entries:
+        source = artifact.source
+        if (await runtime.run(["test", "-e", source], {})).exit_code != 0:
+            if sweep and source == ARTIFACTS_DIR:
+                continue
+            raise RuntimeError(
+                f"declared artifact {source!r} does not exist in the runtime"
+            )
+        archive = await _tar_out(runtime, artifact, budget)
+        budget -= len(archive)
+        collected[source] = archive
+
+    logger.debug("collected artifact roots: %s", list(collected))
+    return collected
+
+
+async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None:
+    """Extract `collected` in `runtime` at the original absolute paths."""
+    if not collected:
+        return
+    # Restoring into the subprocess runtime would extract absolute paths onto the
+    # developer's filesystem, so refuse it before any archive reaches the host.
+    if getattr(runtime.config, "type", None) == "subprocess":
+        raise RuntimeError(
+            "refusing to restore artifacts into the subprocess runtime: extraction "
+            "writes to absolute paths on the host. Grade in a container."
+        )
+    # Clear every root up front, not per entry: a later nested root would otherwise
+    # delete content an earlier one just restored. Clearing also drops any file or
+    # symlink the image left at the target.
+    roots = " ".join(shlex.quote(root) for root in collected)
+    await _run(runtime, f"rm -rf -- {roots}", "clear artifact roots")
+    for root, archive in collected.items():
+        path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar"
+        await runtime.write(path, archive)
+        await _run(
+            runtime,
+            f"tar -xf {shlex.quote(path)} -C / && rm -f {shlex.quote(path)}",
+            f"restore artifact {root!r}",
+        )
+
+
+async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes:
+    path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar"
+    excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude)
+    try:
+        await _run(
+            runtime,
+            f"tar -cf {shlex.quote(path)} -C / {excludes} -- "
+            f"{shlex.quote(artifact.source.lstrip('/'))}",
+            f"collect artifact {artifact.source!r}",
+        )
+        # Size it in the box: an oversized collection is refused before it reaches host
+        # memory, not after.
+        sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {})
+        if (raw := sized.stdout.strip()).isdigit() and int(raw) > budget:
+            raise RuntimeError(
+                f"artifact {artifact.source!r} takes the collection over the "
+                f"{MAX_ARTIFACT_BYTES} byte limit. The grading box boots from the "
+                "agent's image, so only the delta needs to travel — narrow the source "
+                "or add `exclude` patterns."
+            )
+        return await runtime.read(path)
+    finally:
+        # Best-effort: the box is about to be destroyed and the name is unique per call.
+        try:
+            await runtime.run(["rm", "-f", path], {})
+        except Exception:
+            logger.debug("failed to remove %s", path, exc_info=True)
+
+
+async def _run(runtime: Runtime, command: str, action: str) -> None:
+    result = await runtime.run(["sh", "-c", command], {})
+    if result.exit_code:
+        detail = (result.stderr or result.stdout).strip()[-500:]
+        raise RuntimeError(f"failed to {action}: {detail}")

verifiers/v1/cli/dashboard/eval.py

@@ -393,18 +393,19 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     phase_count: dict[str, int] = {}
     model_secs = harness_secs = 0.0
     for trace in done:
-        prompt, completion, cached, reasoning, _ = _tokens(trace)
-        total_in += prompt
-        total_out += completion
-        if cached is not None:
-            total_cached += cached
-            have_cached = True
-        if reasoning is not None:
-            total_reasoning += reasoning
-            have_reasoning = True
-        if trace.usage is not None and trace.usage.cost is not None:
-            total_cost += trace.usage.cost
-            have_cost = True
+        total_in += trace.num_input_tokens
+        total_out += trace.num_output_tokens
+        usage = trace.usage
+        if usage is not None:
+            if usage.cached_input_tokens is not None:
+                total_cached += usage.cached_input_tokens
+                have_cached = True
+            if usage.reasoning_tokens is not None:
+                total_reasoning += usage.reasoning_tokens
+                have_reasoning = True
+            if usage.cost is not None:
+                total_cost += usage.cost
+                have_cost = True
         # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's.
         judge = Usage.aggregate(trace.extra_usage)
         if judge is not None:
@@ -413,13 +414,13 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
             if judge.cost is not None:
                 total_judge_cost += judge.cost
             have_judge = True
-        for phase in ("boot", "setup", "generation", "finalize", "scoring"):
+        for phase in ("boot", "setup", "agent", "finalize", "scoring"):
             span = getattr(trace.timing, phase)
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
                 phase_count[phase] = phase_count.get(phase, 0) + 1
-        model_secs += trace.timing.generation.model.duration
-        harness_secs += trace.timing.generation.harness.duration
+        model_secs += trace.timing.agent.model.duration
+        harness_secs += trace.timing.agent.harness.duration
     if (
         total_in
         or total_out
@@ -448,12 +449,12 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
             usage.append(cost)
         grid.add_row("usage", "  ·  ".join(usage))
     time_segments = []
-    for phase in ("boot", "setup", "generation", "finalize", "scoring"):
+    for phase in ("boot", "setup", "agent", "finalize", "scoring"):
         count = phase_count.get(phase)
         if not count:
             continue
         segment = f"{phase} {format_time(phase_secs[phase] / count)}"
-        if phase == "generation":
+        if phase == "agent":
             segment += (
                 f" (model {format_time(model_secs / count)}"
                 f" + harness {format_time(harness_secs / count)})"
@@ -464,26 +465,6 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     return grid if grid.row_count else None
 
 
-def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
-    """Input/output tokens summed across all branches: per branch, output is every assistant
-    (completion) token generated across its turns and input is the fed-in tokens counted once
-    (system + user + tool) — the final sequence minus everything the model generated. A rollout
-    yields one training sample per branch (a linear trace is a single branch; compaction and
-    subagents add more), so the totals sum them — matching `Trace.num_input_tokens` /
-    `Trace.num_output_tokens`, whose sum is `num_total_tokens`.
-
-    Both counts come from provider-reported usage. Returns the branch count from the same derived
-    view so each dashboard tick materializes it once."""
-    usage = trace.usage
-    cached = usage.cached_input_tokens if usage else None
-    reasoning = usage.reasoning_tokens if usage else None
-    branches = trace.branches
-    nbranches = len(branches)
-    prompt = sum(b.num_input_tokens for b in branches)
-    completion = sum(b.num_output_tokens for b in branches)
-    return prompt, completion, cached, reasoning, nbranches
-
-
 def _stage(trace: Trace) -> str:
     """The stage a live (not-yet-done) rollout is in, derived from its trace's timing
     spans — the engine opens and closes each span exactly at the stage transitions, so
@@ -495,7 +476,7 @@ def _stage(trace: Trace) -> str:
     for stage, span in (
         ("scoring", trace.timing.scoring),
         ("finalize", trace.timing.finalize),
-        ("running", trace.timing.generation),
+        ("running", trace.timing.agent),
         ("setup", trace.timing.setup),
         ("boot", trace.timing.boot),
     ):
@@ -551,7 +532,9 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
             base = f"name={task.name[:32]}" if task.name else f"idx={task.idx}"
             if not slot.traces:
                 if slot.done:  # the env's rollout() itself failed before any trace
-                    error = slot.episode.error if slot.episode is not None else None
+                    error = (
+                        slot.episode.last_error if slot.episode is not None else None
+                    )
                     group_rows.append(
                         (
                             "error",
@@ -602,7 +585,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                 end = (
                     t.timing.scoring.end
                     or t.timing.finalize.end
-                    or t.timing.generation.end
+                    or t.timing.agent.end
                     # a rollout that errored in boot/setup has only that span's end — freeze there
                     # once done, else (still running) the timer would grow off `now` forever
                     or (
@@ -612,8 +595,12 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     )
                     or now
                 )
-                prompt, completion, cached, reasoning, nbranches = _tokens(t)
-                cost = t.usage.cost if t.usage else None
+                prompt, completion = t.num_input_tokens, t.num_output_tokens
+                nbranches = t.num_branches
+                usage = t.usage
+                cached = usage.cached_input_tokens if usage else None
+                reasoning = usage.reasoning_tokens if usage else None
+                cost = usage.cost if usage else None
                 tokens = ""
                 if prompt or completion:
                     tokens = f"{format_count(prompt)}/{format_count(completion)} tokens"

verifiers/v1/cli/debug.py

@@ -123,16 +123,16 @@ def record_debug_error(
     action_timeout: float | None,
 ) -> None:
     now = time.time()
-    for span in (trace.timing.boot, trace.timing.setup, trace.timing.generation):
+    for span in (trace.timing.boot, trace.timing.setup, trace.timing.agent):
         if span.start and not span.end:
             span.end = now
-    in_action = bool(trace.timing.generation.start)
+    in_action = bool(trace.timing.agent.start)
     stage = (
         "debug action" if in_action else "setup" if trace.timing.setup.start else "boot"
     )
     timeout = action_timeout if in_action else setup_timeout
     error_start = (
-        trace.timing.generation.start
+        trace.timing.agent.start
         if in_action
         else trace.timing.setup.start or trace.timing.boot.start
     )
@@ -234,9 +234,9 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         await runtime.prepare_execution([])
         trace.timing.setup.end = time.time()
 
-        trace.timing.generation.start = time.time()
+        trace.timing.agent.start = time.time()
         debug.update(await run_action(runtime, config))
-        trace.timing.generation.end = time.time()
+        trace.timing.agent.end = time.time()
         if not debug.get("ok"):
             record_action_failure(trace, debug)
         trace.stop(str(debug["reason"]))
@@ -246,7 +246,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     except Exception as e:  # noqa: BLE001 - persist any framework failure on the trace
         record_debug_error(trace, debug, e, setup_timeout, config.timeout.total)
     finally:
-        trace.split_generation()
+        trace.split_agent_time()
         trace.info["debug"] = debug
         try:
             await runtime.stop()

verifiers/v1/cli/replay.py

@@ -84,7 +84,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
     episodes = read_episodes(
         source, Trace[WireTaskData, state_cls(task_cls), WireAgentConfig]
     )
-    sourced = [(trace, e.env) for e in episodes for trace in e.traces]
+    sourced = [(trace, e.env.id) for e in episodes for trace in e.traces]
     if config.num_traces is not None:
         sourced = sourced[: config.num_traces]
     traces = [trace for trace, _ in sourced]

verifiers/v1/env.py

@@ -20,7 +20,7 @@ from verifiers.v1.configs.env import (
     _declared_agent_configs,
     default_agent_harness,
 )
-from verifiers.v1.episode import Episode
+from verifiers.v1.episode import EnvInfo, Episode
 from verifiers.v1.errors import EnvError, boundary
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.interception import (
@@ -257,7 +257,7 @@ class Env(ABC, Generic[ConfigT]):
         completed subset, its exception on the episode's `errors`. `on_trace` observes
         each agent-run's trace at mint; `on_discard` its abandonment (a per-agent
         retry mints a replacement)."""
-        episode = Episode(env=self.config.env_id)
+        episode = Episode(env=EnvInfo(id=self.config.env_id))
         agents = self._episode_agents(ctx, episode.traces, on_trace, on_discard)
         try:
             async with asyncio.timeout(self.config.timeout.episode):

verifiers/v1/envs/agentic_judge/__init__.py

@@ -4,6 +4,7 @@ from verifiers.v1.envs.agentic_judge.env import (
     Criterion,
     JudgeTaskConfig,
     ScoreConfig,
+    TextFile,
 )
 
 __all__ = [
@@ -12,4 +13,5 @@ __all__ = [
     "Criterion",
     "JudgeTaskConfig",
     "ScoreConfig",
+    "TextFile",
 ]