verifiers 0.2.2.dev48__py3-none-any.whl → 0.2.2.dev49__py3-none-any.whl

verifiers/v1/agent.py

@@ -45,7 +45,7 @@ from verifiers.v1.types import (
     UserMessage,
 )
 from verifiers.v1.utils.compile import (
-    cap_remote_harness_timeout,
+    cap_remote_agent_timeout,
     resolve_runtime_config,
     validate_pairing,
 )
@@ -390,7 +390,7 @@ class Agent:
                 attempt + 1,
                 retry.max_retries,
                 delay,
-                trace.error.type if trace.error else "?",
+                trace.last_error.type if trace.last_error else "?",
             )
             await asyncio.sleep(delay)
         if history:
@@ -419,8 +419,8 @@ class Agent:
             # close() never runs — free the run's servers and owned runtime first.
             await run.abort()
             raise
-        if trace.runtime is not None:
-            trace.runtime.borrowed = runtime is not None
+        if trace.agent.runtime is not None:
+            trace.agent.runtime.borrowed = runtime is not None
         return trace
 
     @asynccontextmanager
@@ -482,8 +482,8 @@ class Agent:
             opened = await run.open()
             if not opened:
                 trace = await run.close()
-                if trace.runtime is not None:
-                    trace.runtime.borrowed = runtime is not None
+                if trace.agent.runtime is not None:
+                    trace.agent.runtime.borrowed = runtime is not None
         if not opened:
             failure = run.failure
             if failure is None:  # `open()` returning False always captures one.
@@ -499,8 +499,8 @@ class Agent:
             raise
         finally:
             trace = run.trace if run.closed else await interaction.close()
-            if trace.runtime is not None:
-                trace.runtime.borrowed = runtime is not None
+            if trace.agent.runtime is not None:
+                trace.agent.runtime.borrowed = runtime is not None
 
     def _rollout_params(
         self, task: Task, runtime: Runtime | None, shared_tools: dict
@@ -520,10 +520,10 @@ class Agent:
             self.harness, type(task), runtime_config, shared_tools=shared_tools
         )
         # Timeout precedence: agent-level wins, else the task's, else no limit.
-        harness_timeout = (
+        agent_timeout = (
             self.timeout.rollout
             if self.timeout.rollout is not None
-            else task.data.timeout.harness
+            else task.data.timeout.agent
         )
         return {
             "agent_config": self.config,
@@ -535,8 +535,8 @@ class Agent:
                 if self.timeout.setup is not None
                 else task.data.timeout.setup
             ),
-            "harness_timeout": cap_remote_harness_timeout(
-                harness_timeout, runtime_config, task
+            "agent_timeout": cap_remote_agent_timeout(
+                agent_timeout, runtime_config, task
             ),
             "finalize_timeout": (
                 self.timeout.finalize

verifiers/v1/cli/dashboard/eval.py

@@ -302,7 +302,7 @@ def Progress(
     # run, a modeled user) are `trainable=False` and carry no rewards, so counting
     # them dilutes every mean with structural zeros. An all-untrainable run (every
     # role frozen) falls back to all traces rather than showing nothing.
-    scored = [t for t in done_traces if t.trainable] or done_traces
+    scored = [t for t in done_traces if t.agent.trainable] or done_traces
     total = len(slots)
     # Headline reward = mean over non-errored traces; when any errored, `format_mean` appends
     # the global avg (errored count as 0) in parens. `err` is the share of episodes that
@@ -367,13 +367,13 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     # show); usage/time below still cover errored rollouts (their resources were spent regardless).
     has_clean = any(not t.has_error for t in done)
     score_rows = (("rewards", "rewards"), ("metrics", "metrics")) if has_clean else ()
-    by_agent: dict[str | None, list[Trace]] = {}
+    by_agent: dict[str, list[Trace]] = {}
     for trace in done:
-        by_agent.setdefault(trace.agent_name, []).append(trace)
+        by_agent.setdefault(trace.agent.name, []).append(trace)
     for label, source in score_rows:
         if len(by_agent) > 1:
             segments = [
-                f"[dim]{name or '—'}:[/dim] {means}"
+                f"[dim]{name}:[/dim] {means}"
                 for name, traces in by_agent.items()
                 if (means := _score_segments(traces, source)) is not None
             ]
@@ -505,7 +505,7 @@ def _stage(trace: Trace) -> str:
         stage = "boot"  # trace minted, first span not yet opened (an instant)
     # A boot stuck on a first-use platform image build reads differently from a
     # normal boot — it can sit there for ~10 minutes (prime runtime only).
-    if stage == "boot" and getattr(trace.runtime, "image_cached", None) is False:
+    if stage == "boot" and getattr(trace.agent.runtime, "image_cached", None) is False:
         return "build"
     return stage
 
@@ -564,14 +564,14 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     group_rows.append(("pending", [f"task {base}", *[""] * 7], "", ""))
                 continue
             for t in slot.traces:
-                label = f"{base} agent={t.agent_name}" if t.agent_name else base
+                label = f"{base} agent={t.agent.name}"
                 if slot.done:  # fully scored — reward is final
                     state = "error" if t.has_error else "success"
                     # A trace that recorded nothing shows no reward: a judge or
                     # modeled-user seat's `reward=0.00` would read as a score.
                     result = (
-                        t.error.type
-                        if t.has_error
+                        t.last_error.type
+                        if t.has_error and t.last_error
                         else (f"reward={t.reward:.2f}" if t.rewards else "")
                     )
                     if t.has_error:
@@ -582,7 +582,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                             t.is_truncated
                         ):  # flag a clipped rollout next to its stop condition
                             stop = f"{stop} (truncated)".strip()
-                elif t.is_completed and (err := t.error) is not None:
+                elif t.is_completed and (err := t.last_error) is not None:
                     # An errored trace whose episode is still running its other
                     # traces (or `score()`) is already a failure — show it, don't
                     # let it sit as "scoring" until the whole episode lands.
@@ -592,12 +592,9 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                 # The trace's own stamp, not the run-level runtime: a role's harness
                 # may resolve elsewhere (the judge env's sandboxed judge on a
                 # subprocess run).
-                if t.runtime is not None:
-                    runtime = (
-                        f"{t.runtime.type}({t.runtime.id})"
-                        if t.runtime.id
-                        else t.runtime.type
-                    )
+                if t.agent.runtime is not None:
+                    rt = t.agent.runtime
+                    runtime = f"{rt.type}({rt.id})" if rt.id else rt.type
                 else:
                     runtime = runtime_type
                 turns = t.num_turns
@@ -635,7 +632,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     f"{nbranches} branch{'es' * (nbranches != 1)}",
                     tokens,
                     f"{format_cost_usd(cost)}" if cost is not None else "",
-                    stop,  # stop condition (agent_completed / max_turns / harness_timeout), once done
+                    stop,  # stop condition (agent_completed / max_turns / error), once done
                 ]
                 # No start time yet (queued, not generating) → blank, not `now - 0` (~56 years).
                 elapsed = format_time(end - start) if start else ""

verifiers/v1/cli/debug.py

@@ -101,9 +101,9 @@ def error_info(
 
 
 def capture_trace_error(trace: Trace, error: BaseException) -> None:
-    # CancelledError is a BaseException; Trace.capture_error accepts Exception.
+    # CancelledError is a BaseException; Trace.record_error accepts Exception.
     if isinstance(error, Exception):
-        trace.capture_error(error)
+        trace.record_error(error)
         return
     trace.errors.append(
         Error(

verifiers/v1/cli/eval/runner.py

@@ -68,7 +68,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
 
     async def on_complete(episode: Episode) -> None:
         for trace in episode.traces:
-            trace.stamp(EvalRunInfo(id=config.uuid))
+            trace.record_run(EvalRunInfo(id=config.uuid))
         await append_episode(out, episode, write_lock)
 
     # Serving resources (shared tool servers, interception) come up once for the
@@ -240,7 +240,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 )
             records = []
             for trace in traces:
-                trace.stamp(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.uuid))
                 await append_trace(out, trace, write_lock, env=config.env_id)
                 records.append(Episode.of(trace))
             return records
@@ -254,7 +254,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     **payload,
                 )
             for trace in episode.traces:
-                trace.stamp(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.uuid))
             await append_episode(out, episode, write_lock)
             return [episode]
 

verifiers/v1/cli/replay.py

@@ -165,7 +165,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
                     st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
                 except Exception as exc:
                     st.state, st.detail = "error", type(exc).__name__
-                    trace.capture_error(exc)
+                    trace.record_error(exc)
                     if not config.rich:
                         logger.warning(
                             "replay: scoring failed for task %s",

verifiers/v1/cli/validate.py

@@ -89,6 +89,12 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
         trace = Trace(
             task=TraceTask(type=type(task).__name__, data=task.data),
             state=state_cls(type(task))(),
+            # No agent runs here — the info only records the runtime policy.
+            agent=vf.AgentInfo(
+                config=vf.AgentConfig(runtime=config.runtime),
+                name="validate",
+                trainable=False,
+            ),
         )
         await runtime.start()
         await asyncio.wait_for(
@@ -124,6 +130,12 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
         trace = Trace(
             task=TraceTask(type=type(task).__name__, data=task.data),
             state=state_cls(type(task))(),
+            # No agent runs here — the info only records the runtime policy.
+            agent=vf.AgentInfo(
+                config=vf.AgentConfig(runtime=config.runtime),
+                name="validate",
+                trainable=False,
+            ),
         )
         await runtime.start()
         await asyncio.wait_for(

verifiers/v1/env.py

@@ -163,7 +163,7 @@ class Env(ABC, Generic[ConfigT]):
         """Cross-agent judgement — THE programmable judgement surface: plain
         imperative Python over the finished episode (per-trace judgement already
         ran on each trace's own task). `episode.traces` is the flat episode in
-        completion order, each trace's `agent_name` stamp naming its agent; attach
+        completion order, each trace's `agent.name` stamp naming its agent; attach
         signals via `record_reward`/`record_metric`, in program order. A raise
         fails the episode (the retryable unit) — validate strictly, never
         record a guess."""

verifiers/v1/envs/agentic_judge/env.py

@@ -304,7 +304,7 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
             await agents.judge.run(judge_task, runtime=box)
 
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
-        by_agent = {t.agent_name: t for t in episode.traces}
+        by_agent = {t.agent.name: t for t in episode.traces}
         solution, verdict = by_agent["solver"], by_agent["judge"]
         data = verdict.info.get("verdict")
         if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):

verifiers/v1/envs/user_sim/env.py

@@ -89,7 +89,7 @@ class UserSimEnv(vf.Env[UserSimEnvConfig]):
     async def finalize(self, task, episode):
         """One conversation-shape fact about the user's side, recorded on the
         assistant's trace; judgement stays on the task's rewards."""
-        (user,) = (t for t in episode.traces if t.agent_name == "user")
+        (user,) = (t for t in episode.traces if t.agent.name == "user")
         for trace in episode.traces:
-            if trace.agent_name == "assistant":
+            if trace.agent.name == "assistant":
                 trace.record_metric("user_turns", float(user.num_turns))

verifiers/v1/errors.py

@@ -4,7 +4,7 @@ Four mechanisms, each in one place:
 
 1. Vocabulary (this module): `RolloutError` and the flat boundary types below. Each names the
    boundary a failure crossed — provider, harness, toolset, sandbox, task, or
-   interception — so a recorded `trace.error.type` says where the rollout broke.
+   interception — so a recorded `trace.last_error.type` says where the rollout broke.
 2. Classification (`boundary`): the one helper that runs a framework→code boundary and attributes
    any escaping error to that boundary's type. Extension code (task hooks, harness subclasses)
    raises plain Python errors — it never constructs a `vf` error type; `boundary` classifies them.

verifiers/v1/gepa/adapter.py

@@ -104,10 +104,9 @@ class GEPAAdapter:
                     "completion": trace.last_reply,
                     "reward": trace.reward,
                 }
-                if trace.agent_name:
-                    record["agent"] = trace.agent_name
+                record["agent"] = trace.agent.name
                 if trace.has_error:
-                    record["error"] = str(trace.error)
+                    record["error"] = str(trace.last_error)
                 if trace.stop_condition:
                     record["stop_condition"] = trace.stop_condition
                 for column in self.reflection_columns:

verifiers/v1/legacy.py

@@ -24,6 +24,7 @@ from pydantic import ValidationError
 
 from verifiers.v1 import graph
 from verifiers.v1.clients.config import ClientConfig, TrainClientConfig
+from verifiers.v1.configs.agent import AgentConfig
 from verifiers.v1.episode import Episode
 from verifiers.v1.serve.server import EnvServer
 from verifiers.v1.serve.types import (
@@ -34,6 +35,7 @@ from verifiers.v1.serve.types import (
 )
 from verifiers.v1.task import WireTaskData
 from verifiers.v1.trace import (
+    AgentInfo,
     Error,
     GenerationSpan,
     ModelCall,
@@ -227,7 +229,6 @@ def _timing(raw: Any) -> Timing:
 _V0_TO_V1_TRUNCATION_STOP = {
     "max_turns_reached": "max_turns",
     "prompt_too_long": "context_length",
-    "timeout_reached": "harness_timeout",
     "max_total_completion_tokens_reached": "max_output_tokens",
 }
 
@@ -270,7 +271,10 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             type="Task",
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
-        tools=_to_v1_tools(out.get("tool_defs")),
+        # v0 rollouts carry no agent config — record the default, like the
+        # base task type above.
+        agent=AgentInfo(config=AgentConfig()),
+        tools=_to_v1_tools(out.get("tool_defs")) or [],
         rewards={"reward": Reward(score=float(out.get("reward") or 0.0))},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         info=dict(out.get("info") or {}),

verifiers/v1/push.py

@@ -58,8 +58,8 @@ def trace_to_sample(
         "example_id": trace.task.data.idx,
         "rollout_number": rollout_number,
         "episode_id": episode_id,
-        "agent": trace.agent_name,
-        "trainable": trace.trainable,
+        "agent": trace.agent.name,
+        "trainable": trace.agent.trainable,
         "task": task,
         "prompt": [],
         "completion": dump(branches[-1].messages) if branches else [],
@@ -73,8 +73,8 @@ def trace_to_sample(
         "is_completed": trace.is_completed,
         "is_truncated": trace.is_truncated,
         "metrics": trace.metrics,
-        "error": trace.error.model_dump(mode="json", exclude_none=True)
-        if trace.error
+        "error": trace.last_error.model_dump(mode="json", exclude_none=True)
+        if trace.last_error
         else None,
         "stop_condition": trace.stop_condition,
         "trajectory": [
@@ -124,7 +124,7 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     back to all traces when none are trainable (same rule as the dashboard).
     `avg_error` is the share of EPISODES that aren't ok: a hook failure counts
     even when its traces are clean or it left none."""
-    scored = [t for t in traces if t.trainable] or traces
+    scored = [t for t in traces if t.agent.trainable] or traces
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:

verifiers/v1/retries.py

@@ -117,7 +117,9 @@ async def run_episode_with_retry(
         final = await run()
         if attempt == retry.max_retries or not episode_should_retry(final, retry):
             break
-        cause = final.error or next((t.error for t in final.traces if t.error), None)
+        cause = final.error or next(
+            (t.last_error for t in final.traces if t.last_error), None
+        )
         history.extend(final.errors)
         for trace in final.traces:
             history.extend(trace.errors)

verifiers/v1/rollout.py

@@ -24,7 +24,6 @@ import time
 from collections.abc import AsyncIterator, Callable
 from contextlib import AsyncExitStack, asynccontextmanager
 
-from verifiers import __version__
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.agent import AgentConfig
 from verifiers.v1.decorators import discover_decorated, invoke
@@ -52,9 +51,8 @@ from verifiers.v1.runtimes import (
 from verifiers.v1.session import RolloutLimits, RolloutSession
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task, TaskData
-from verifiers.v1.trace import AgentInfo, Trace, TraceTask, VersionInfo
+from verifiers.v1.trace import AgentInfo, Trace, TraceTask
 from verifiers.v1.types import Messages
-from verifiers.v1.utils.version import verifiers_commit
 
 logger = logging.getLogger(__name__)
 
@@ -118,7 +116,7 @@ class RolloutRun:
         wire_data: TaskData | None = None,
         has_user: bool = False,
         setup_timeout: float | None = None,
-        harness_timeout: float | None = None,
+        agent_timeout: float | None = None,
         finalize_timeout: float | None = None,
         scoring_timeout: float | None = None,
         limits: RolloutLimits | None = None,
@@ -133,7 +131,8 @@ class RolloutRun:
         self.runtime_config = runtime_config
         self._has_user = has_user
         self._setup_timeout = setup_timeout
-        self._harness_time_remaining = harness_timeout
+        self._agent_timeout = agent_timeout
+        self._agent_time_remaining = agent_timeout
         self._finalize_timeout = finalize_timeout
         self._scoring_timeout = scoring_timeout
         self._shared_tools = shared_tools or {}
@@ -146,7 +145,6 @@ class RolloutRun:
                 data=task.data if wire_data is None else wire_data,
             ),
             state=state_cls(type(task))(),
-            verifiers=VersionInfo(version=__version__, commit=verifiers_commit()),
             # The seat's resolved config, role overrides included — the agent
             # this trace can be reproduced with.
             agent=AgentInfo(config=agent_config),
@@ -166,7 +164,7 @@ class RolloutRun:
         self.deadline_at: float | None = None
         """The active harness segment's absolute deadline (event-loop clock), or
         None between segments / when unbounded. An interaction spends one cumulative
-        `harness_timeout` budget only while its own segments run, so time awaiting
+        `agent_timeout` budget only while its own segments run, so time awaiting
         the caller (including another interleaved agent) cannot starve it."""
 
     @property
@@ -201,7 +199,7 @@ class RolloutRun:
             logger.exception("unexpected error in rollout %s", self.trace.id)
         self._failed = True
         self._failure = error
-        self.trace.capture_error(error)
+        self.trace.record_error(error)
 
     async def open(self) -> bool:
         """Boot the rollout's world up to the point where segments can run: start
@@ -308,7 +306,8 @@ class RolloutRun:
         for an exchange the user opens, this is also the first segment, on an
         empty conversation); without, it launches on the task's own prompt.
         Returns whether the exchange can continue — a refused turn (limit, @stop),
-        a timeout, a failure, or a segment that made no progress all end it."""
+        a failure (an expired agent timeout included), or a segment that made no
+        progress all end it."""
         if not self._opened or self._closed or not self.ok:
             return False
         trace = self.trace
@@ -317,11 +316,10 @@ class RolloutRun:
         segment_start = loop.time()
         self.deadline_at = (
             None
-            if self._harness_time_remaining is None
-            else segment_start + max(0.0, self._harness_time_remaining)
+            if self._agent_time_remaining is None
+            else segment_start + max(0.0, self._agent_time_remaining)
         )
         # Prefer an intercepted model/tool error to the harness exit it caused.
-        # A timeout still scores the partial trajectory.
         try:
             async with asyncio.timeout_at(self.deadline_at):
                 await self.harness.run(
@@ -335,11 +333,16 @@ class RolloutRun:
                     messages,
                 )
         except TimeoutError as e:
-            # Only the rollout deadline reads as a clean truncation; a TimeoutError
-            # from the harness's own I/O with no expired deadline is a failure —
-            # recording it as a stop would score a broken run as a partial success.
+            # An expired rollout deadline is the agent breaking its time budget —
+            # an agent failure, never a clean stop. A TimeoutError from the
+            # harness's own I/O with no expired deadline stays the raw failure.
             if self.deadline_at is not None and (loop.time() >= self.deadline_at):
-                trace.stop("harness_timeout")
+                self.fail(
+                    HarnessError(
+                        f"agent timeout: rollout exceeded its "
+                        f"{self._agent_timeout:g}s budget"
+                    )
+                )
             else:
                 self.fail(e)
             return False
@@ -352,9 +355,9 @@ class RolloutRun:
                 self.fail(e)
             return False
         finally:
-            if self._harness_time_remaining is not None:
-                self._harness_time_remaining = max(
-                    0.0, self._harness_time_remaining - (loop.time() - segment_start)
+            if self._agent_time_remaining is not None:
+                self._agent_time_remaining = max(
+                    0.0, self._agent_time_remaining - (loop.time() - segment_start)
                 )
             self.deadline_at = None
         if self._session.error is not None:
@@ -457,6 +460,6 @@ class RolloutRun:
             self.task.data.idx,
             trace.reward,
             trace.num_turns,
-            trace.error.type if trace.error else trace.stop_condition,
+            trace.last_error.type if trace.last_error else trace.stop_condition,
         )
         return trace

verifiers/v1/runtimes/subprocess.py

@@ -62,7 +62,7 @@ class SubprocessRuntime(Runtime):
             stdout, stderr = await proc.communicate()
         finally:
             # If the await didn't finish, the caller cancelled it (e.g. the rollout's
-            # scoring_timeout / harness_timeout fired): communicate() leaves the process
+            # scoring_timeout / agent_timeout fired): communicate() leaves the process
             # running, so SIGKILL its whole group (start_new_session => pgid == pid) — otherwise
             # a hung child (a wedged uv/sympy verify) outlives the rollout and leaks CPU. A
             # no-op once it has exited on its own.

verifiers/v1/task.py

@@ -89,7 +89,7 @@ class TaskTimeout(StrictBaseModel):
     model_config = ConfigDict(frozen=True)
 
     setup: float | None = None
-    harness: float | None = None
+    agent: float | None = None
     finalize: float | None = None
     scoring: float | None = None
 

verifiers/v1/tasksets/harbor/taskset.py

@@ -260,9 +260,9 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
     if not authors and meta.get("author_name"):
         authors = [Author(name=meta["author_name"], email=meta.get("author_email"))]
     if harbor_config.ignore_timeouts:
-        harness_timeout = scoring_timeout = None
+        agent_timeout = scoring_timeout = None
     else:
-        harness_timeout = (
+        agent_timeout = (
             parsed.agent.timeout_sec
             if "timeout_sec" in parsed.agent.model_fields_set
             else None
@@ -290,8 +290,8 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
             else list(network.allowed_hosts)
         ),
         timeout=TaskTimeout(
-            harness=harness_timeout * harbor_config.timeout_multiplier
-            if harness_timeout is not None
+            agent=agent_timeout * harbor_config.timeout_multiplier
+            if agent_timeout is not None
             else None,
             scoring=scoring_timeout * harbor_config.timeout_multiplier
             if scoring_timeout is not None