verifiers 0.2.2.dev48__py3-none-any.whl → 0.2.2.dev50__py3-none-any.whl

verifiers/v1/agent.py

@@ -45,7 +45,7 @@ from verifiers.v1.types import (
     UserMessage,
 )
 from verifiers.v1.utils.compile import (
-    cap_remote_harness_timeout,
+    cap_remote_agent_timeout,
     resolve_runtime_config,
     validate_pairing,
 )
@@ -390,7 +390,7 @@ class Agent:
                 attempt + 1,
                 retry.max_retries,
                 delay,
-                trace.error.type if trace.error else "?",
+                trace.last_error.type if trace.last_error else "?",
             )
             await asyncio.sleep(delay)
         if history:
@@ -419,8 +419,8 @@ class Agent:
             # close() never runs — free the run's servers and owned runtime first.
             await run.abort()
             raise
-        if trace.runtime is not None:
-            trace.runtime.borrowed = runtime is not None
+        if trace.agent.runtime is not None:
+            trace.agent.runtime.borrowed = runtime is not None
         return trace
 
     @asynccontextmanager
@@ -482,8 +482,8 @@ class Agent:
             opened = await run.open()
             if not opened:
                 trace = await run.close()
-                if trace.runtime is not None:
-                    trace.runtime.borrowed = runtime is not None
+                if trace.agent.runtime is not None:
+                    trace.agent.runtime.borrowed = runtime is not None
         if not opened:
             failure = run.failure
             if failure is None:  # `open()` returning False always captures one.
@@ -499,8 +499,8 @@ class Agent:
             raise
         finally:
             trace = run.trace if run.closed else await interaction.close()
-            if trace.runtime is not None:
-                trace.runtime.borrowed = runtime is not None
+            if trace.agent.runtime is not None:
+                trace.agent.runtime.borrowed = runtime is not None
 
     def _rollout_params(
         self, task: Task, runtime: Runtime | None, shared_tools: dict
@@ -520,10 +520,10 @@ class Agent:
             self.harness, type(task), runtime_config, shared_tools=shared_tools
         )
         # Timeout precedence: agent-level wins, else the task's, else no limit.
-        harness_timeout = (
+        agent_timeout = (
             self.timeout.rollout
             if self.timeout.rollout is not None
-            else task.data.timeout.harness
+            else task.data.timeout.agent
         )
         return {
             "agent_config": self.config,
@@ -535,8 +535,8 @@ class Agent:
                 if self.timeout.setup is not None
                 else task.data.timeout.setup
             ),
-            "harness_timeout": cap_remote_harness_timeout(
-                harness_timeout, runtime_config, task
+            "agent_timeout": cap_remote_agent_timeout(
+                agent_timeout, runtime_config, task
             ),
             "finalize_timeout": (
                 self.timeout.finalize

verifiers/v1/cli/dashboard/eval.py

@@ -302,7 +302,7 @@ def Progress(
     # run, a modeled user) are `trainable=False` and carry no rewards, so counting
     # them dilutes every mean with structural zeros. An all-untrainable run (every
     # role frozen) falls back to all traces rather than showing nothing.
-    scored = [t for t in done_traces if t.trainable] or done_traces
+    scored = [t for t in done_traces if t.agent.trainable] or done_traces
     total = len(slots)
     # Headline reward = mean over non-errored traces; when any errored, `format_mean` appends
     # the global avg (errored count as 0) in parens. `err` is the share of episodes that
@@ -367,13 +367,13 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     # show); usage/time below still cover errored rollouts (their resources were spent regardless).
     has_clean = any(not t.has_error for t in done)
     score_rows = (("rewards", "rewards"), ("metrics", "metrics")) if has_clean else ()
-    by_agent: dict[str | None, list[Trace]] = {}
+    by_agent: dict[str, list[Trace]] = {}
     for trace in done:
-        by_agent.setdefault(trace.agent_name, []).append(trace)
+        by_agent.setdefault(trace.agent.name, []).append(trace)
     for label, source in score_rows:
         if len(by_agent) > 1:
             segments = [
-                f"[dim]{name or '—'}:[/dim] {means}"
+                f"[dim]{name}:[/dim] {means}"
                 for name, traces in by_agent.items()
                 if (means := _score_segments(traces, source)) is not None
             ]
@@ -505,7 +505,7 @@ def _stage(trace: Trace) -> str:
         stage = "boot"  # trace minted, first span not yet opened (an instant)
     # A boot stuck on a first-use platform image build reads differently from a
     # normal boot — it can sit there for ~10 minutes (prime runtime only).
-    if stage == "boot" and getattr(trace.runtime, "image_cached", None) is False:
+    if stage == "boot" and getattr(trace.agent.runtime, "image_cached", None) is False:
         return "build"
     return stage
 
@@ -564,14 +564,14 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     group_rows.append(("pending", [f"task {base}", *[""] * 7], "", ""))
                 continue
             for t in slot.traces:
-                label = f"{base} agent={t.agent_name}" if t.agent_name else base
+                label = f"{base} agent={t.agent.name}"
                 if slot.done:  # fully scored — reward is final
                     state = "error" if t.has_error else "success"
                     # A trace that recorded nothing shows no reward: a judge or
                     # modeled-user seat's `reward=0.00` would read as a score.
                     result = (
-                        t.error.type
-                        if t.has_error
+                        t.last_error.type
+                        if t.has_error and t.last_error
                         else (f"reward={t.reward:.2f}" if t.rewards else "")
                     )
                     if t.has_error:
@@ -582,7 +582,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                             t.is_truncated
                         ):  # flag a clipped rollout next to its stop condition
                             stop = f"{stop} (truncated)".strip()
-                elif t.is_completed and (err := t.error) is not None:
+                elif t.is_completed and (err := t.last_error) is not None:
                     # An errored trace whose episode is still running its other
                     # traces (or `score()`) is already a failure — show it, don't
                     # let it sit as "scoring" until the whole episode lands.
@@ -592,12 +592,9 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                 # The trace's own stamp, not the run-level runtime: a role's harness
                 # may resolve elsewhere (the judge env's sandboxed judge on a
                 # subprocess run).
-                if t.runtime is not None:
-                    runtime = (
-                        f"{t.runtime.type}({t.runtime.id})"
-                        if t.runtime.id
-                        else t.runtime.type
-                    )
+                if t.agent.runtime is not None:
+                    rt = t.agent.runtime
+                    runtime = f"{rt.type}({rt.id})" if rt.id else rt.type
                 else:
                     runtime = runtime_type
                 turns = t.num_turns
@@ -635,7 +632,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     f"{nbranches} branch{'es' * (nbranches != 1)}",
                     tokens,
                     f"{format_cost_usd(cost)}" if cost is not None else "",
-                    stop,  # stop condition (agent_completed / max_turns / harness_timeout), once done
+                    stop,  # stop condition (agent_completed / max_turns / error), once done
                 ]
                 # No start time yet (queued, not generating) → blank, not `now - 0` (~56 years).
                 elapsed = format_time(end - start) if start else ""

verifiers/v1/cli/debug.py

@@ -101,9 +101,9 @@ def error_info(
 
 
 def capture_trace_error(trace: Trace, error: BaseException) -> None:
-    # CancelledError is a BaseException; Trace.capture_error accepts Exception.
+    # CancelledError is a BaseException; Trace.record_error accepts Exception.
     if isinstance(error, Exception):
-        trace.capture_error(error)
+        trace.record_error(error)
         return
     trace.errors.append(
         Error(

verifiers/v1/cli/eval/runner.py

@@ -68,7 +68,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
 
     async def on_complete(episode: Episode) -> None:
         for trace in episode.traces:
-            trace.stamp(EvalRunInfo(id=config.uuid))
+            trace.record_run(EvalRunInfo(id=config.uuid))
         await append_episode(out, episode, write_lock)
 
     # Serving resources (shared tool servers, interception) come up once for the
@@ -240,7 +240,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 )
             records = []
             for trace in traces:
-                trace.stamp(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.uuid))
                 await append_trace(out, trace, write_lock, env=config.env_id)
                 records.append(Episode.of(trace))
             return records
@@ -254,7 +254,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     **payload,
                 )
             for trace in episode.traces:
-                trace.stamp(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.uuid))
             await append_episode(out, episode, write_lock)
             return [episode]
 

verifiers/v1/cli/replay.py

@@ -165,7 +165,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
                     st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
                 except Exception as exc:
                     st.state, st.detail = "error", type(exc).__name__
-                    trace.capture_error(exc)
+                    trace.record_error(exc)
                     if not config.rich:
                         logger.warning(
                             "replay: scoring failed for task %s",

verifiers/v1/cli/validate.py

@@ -89,6 +89,12 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
         trace = Trace(
             task=TraceTask(type=type(task).__name__, data=task.data),
             state=state_cls(type(task))(),
+            # No agent runs here — the info only records the runtime policy.
+            agent=vf.AgentInfo(
+                config=vf.AgentConfig(runtime=config.runtime),
+                name="validate",
+                trainable=False,
+            ),
         )
         await runtime.start()
         await asyncio.wait_for(
@@ -124,6 +130,12 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
         trace = Trace(
             task=TraceTask(type=type(task).__name__, data=task.data),
             state=state_cls(type(task))(),
+            # No agent runs here — the info only records the runtime policy.
+            agent=vf.AgentInfo(
+                config=vf.AgentConfig(runtime=config.runtime),
+                name="validate",
+                trainable=False,
+            ),
         )
         await runtime.start()
         await asyncio.wait_for(

verifiers/v1/configs/judge.py

@@ -5,7 +5,7 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
-from pydantic import BaseModel, SerializeAsAny, model_validator
+from pydantic import BaseModel, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
@@ -20,15 +20,8 @@ class JudgeConfig(BaseClientConfig):
     weight: float = 1.0
     model: str = "openai/gpt-5.4-nano"
     sampling: SamplingConfig = SamplingConfig()
-    prompt: str | None = None
-    prompt_file: Path | None = None
-    """Prompt file override, mutually exclusive with `prompt`."""
-
-    @model_validator(mode="after")
-    def check_prompt_source(self) -> "JudgeConfig":
-        if self.prompt is not None and self.prompt_file is not None:
-            raise ValueError("set `prompt` or `prompt_file`, not both")
-        return self
+    prompt: Path | None = None
+    """File whose text overrides the judge's default prompt template."""
 
 
 Judges = list[SerializeAsAny[JudgeConfig]]

verifiers/v1/configs/taskset.py

@@ -1,5 +1,7 @@
 """The taskset plugin's config: which rows load, under `--env.taskset.*`."""
 
+from pathlib import Path
+
 from pydantic import SerializeAsAny
 from pydantic_config import BaseConfig
 
@@ -14,6 +16,9 @@ class TasksetConfig(BaseConfig):
     positional `eval <taskset-id>`)."""
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
+    system_prompt: Path | None = None
+    """File whose text overrides each task's `TaskData.system_prompt` in
+    `Taskset.select` (e.g. a GEPA `best_system_prompt.txt`)."""
 
     @property
     def name(self) -> str:

verifiers/v1/env.py

@@ -163,7 +163,7 @@ class Env(ABC, Generic[ConfigT]):
         """Cross-agent judgement — THE programmable judgement surface: plain
         imperative Python over the finished episode (per-trace judgement already
         ran on each trace's own task). `episode.traces` is the flat episode in
-        completion order, each trace's `agent_name` stamp naming its agent; attach
+        completion order, each trace's `agent.name` stamp naming its agent; attach
         signals via `record_reward`/`record_metric`, in program order. A raise
         fails the episode (the retryable unit) — validate strictly, never
         record a guess."""

verifiers/v1/envs/agentic_judge/env.py

@@ -180,41 +180,31 @@ class JudgeTask(vf.Task):
 class JudgeTaskConfig(vf.BaseConfig):
     """The judge's minted task: the grading policy and what lands in its box."""
 
-    prompt: Path | str | None = None
-    """Grading-policy override: inline text, or a policy file (a value ending in
-    `.md`/`.txt` is read from disk). Replaces only the policy body — the verdict
-    contract and workspace note are always appended, so a custom policy cannot
-    break verdict scraping. May reference `{prompt}` (the solver task's prompt);
-    if it doesn't, the task statement is appended after the policy."""
-    hint: Path | str | None = None
-    """Optional hints injected as their own section (inline text or a `.md`/
-    `.txt` file): task-family pointers into the trace or box — e.g. for math,
-    where the reference answer lives in the record; for SWE, to diff the repo
-    or read `info.patch`."""
+    prompt: Path | None = None
+    """Grading-policy file. Replaces only the policy body — the verdict contract
+    and workspace note are always appended. May reference `{prompt}` (the solver
+    task's prompt); if it doesn't, the task statement is appended after."""
+    hint: Path | None = None
+    """Optional hints file injected as their own section: task-family pointers
+    into the trace or box — e.g. for math, where the reference answer lives in
+    the record; for SWE, to diff the repo or read `info.patch`."""
     rubric: Path | None = None
     """Criteria the judge grades against: a `.toml`/`.json` file with a
     `criteria` list — the plugged rubric judge's format, so the same rubric
     files work for both. None grades the single built-in `solved` criterion."""
 
-    @staticmethod
-    def _resolve(value: Path | str) -> str:
-        path = Path(value)
-        if isinstance(value, Path) or path.suffix in (".md", ".txt"):
-            return path.read_text(encoding="utf-8")
-        return str(value)
-
     def build_prompt(self) -> str:
         if self.prompt is None:
             return GRADE_PROMPT + "\n\n" + TASK_SECTION
-        return self._resolve(self.prompt)
+        return self.prompt.read_text()
 
     def build_hint(self) -> str | None:
-        return self._resolve(self.hint) if self.hint is not None else None
+        return self.hint.read_text() if self.hint is not None else None
 
     def criteria(self) -> list[Criterion]:
         if self.rubric is None:
             return [SOLVED]
-        text = self.rubric.read_text(encoding="utf-8")
+        text = self.rubric.read_text()
         data = (
             tomllib.loads(text)
             if self.rubric.suffix.lower() == ".toml"
@@ -304,7 +294,7 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
             await agents.judge.run(judge_task, runtime=box)
 
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
-        by_agent = {t.agent_name: t for t in episode.traces}
+        by_agent = {t.agent.name: t for t in episode.traces}
         solution, verdict = by_agent["solver"], by_agent["judge"]
         data = verdict.info.get("verdict")
         if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):

verifiers/v1/envs/user_sim/env.py

@@ -89,7 +89,7 @@ class UserSimEnv(vf.Env[UserSimEnvConfig]):
     async def finalize(self, task, episode):
         """One conversation-shape fact about the user's side, recorded on the
         assistant's trace; judgement stays on the task's rewards."""
-        (user,) = (t for t in episode.traces if t.agent_name == "user")
+        (user,) = (t for t in episode.traces if t.agent.name == "user")
         for trace in episode.traces:
-            if trace.agent_name == "assistant":
+            if trace.agent.name == "assistant":
                 trace.record_metric("user_turns", float(user.num_turns))

verifiers/v1/errors.py

@@ -4,7 +4,7 @@ Four mechanisms, each in one place:
 
 1. Vocabulary (this module): `RolloutError` and the flat boundary types below. Each names the
    boundary a failure crossed — provider, harness, toolset, sandbox, task, or
-   interception — so a recorded `trace.error.type` says where the rollout broke.
+   interception — so a recorded `trace.last_error.type` says where the rollout broke.
 2. Classification (`boundary`): the one helper that runs a framework→code boundary and attributes
    any escaping error to that boundary's type. Extension code (task hooks, harness subclasses)
    raises plain Python errors — it never constructs a `vf` error type; `boundary` classifies them.

verifiers/v1/gepa/adapter.py

@@ -68,14 +68,10 @@ class GEPAAdapter:
         )
 
     async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Episode]:
-        # Inject the candidate by rebuilding each Task around a data row with the new
-        # system_prompt (TaskData is frozen; behavior/config carry over unchanged).
-        tasks = [
-            type(t)(
-                t.data.model_copy(update={"system_prompt": system_prompt}), t.config
-            )
-            for t in (self.tasks[idx] for idx in batch)
-        ]
+        # Inject the candidate as a copy of each base task with its system_prompt overridden
+        # (`with_system_prompt` copies rather than reconstructs, so subclass state survives and
+        # the shared base task in `self.tasks` is left untouched for the next candidate).
+        tasks = [self.tasks[idx].with_system_prompt(system_prompt) for idx in batch]
         slots = [slot for task in tasks for slot in self.env.slots(task)]
         results = await asyncio.gather(
             *(
@@ -104,10 +100,9 @@ class GEPAAdapter:
                     "completion": trace.last_reply,
                     "reward": trace.reward,
                 }
-                if trace.agent_name:
-                    record["agent"] = trace.agent_name
+                record["agent"] = trace.agent.name
                 if trace.has_error:
-                    record["error"] = str(trace.error)
+                    record["error"] = str(trace.last_error)
                 if trace.stop_condition:
                     record["stop_condition"] = trace.stop_condition
                 for column in self.reflection_columns:

verifiers/v1/gepa/runner.py

@@ -105,7 +105,20 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
                 "skip_perfect_score": False,
                 "logger": _GEPALog(),
             }
-            return optimize(**optimize_kwargs)
+            result = optimize(**optimize_kwargs)
+            if run_dir is not None:
+                # Persist the winning prompt as a plain file so it can be handed straight to
+                # eval/train via `--env.taskset.system-prompt` (see TasksetConfig).
+                candidate = result.best_candidate
+                best = (
+                    candidate.get("system_prompt", "")
+                    if isinstance(candidate, dict)
+                    else str(candidate)
+                )
+                best_path = run_dir / "best_system_prompt.txt"
+                best_path.write_text(best, encoding="utf-8")
+                logger.info("best system prompt: %s", best_path)
+            return result
         finally:
             loop.run_until_complete(serving.__aexit__(None, None, None))
     finally:

verifiers/v1/judge.py

@@ -116,13 +116,13 @@ def judge_config_cls(cls: type) -> type[JudgeConfig]:
 
 class Judge(Generic[ParsedT, ConfigT]):
     prompt: str | None = None
-    """Default prompt template, overridden by config."""
+    """Default prompt template, overridden by a config `prompt` file."""
     schema: type[BaseModel] | None = None
 
     def __init__(self, config: ConfigT | None = None) -> None:
         self.config = cast(ConfigT, config or judge_config_cls(type(self))())
-        if self.config.prompt_file is not None:
-            self.prompt = self.config.prompt_file.read_text(encoding="utf-8")
+        if self.config.prompt is not None:
+            self.prompt = self.config.prompt.read_text()
 
     @property
     def reward_name(self) -> str:
@@ -132,7 +132,7 @@ class Judge(Generic[ParsedT, ConfigT]):
         return judge_key(self.config) or fallback or "judge"
 
     def build_messages(self, **fields: Any) -> str | Messages:
-        template = self.config.prompt or self.prompt
+        template = self.prompt
         if template is None:
             raise ValueError(
                 f"{type(self).__name__} has no `prompt`; set it or override build_messages"

verifiers/v1/legacy.py

@@ -24,6 +24,7 @@ from pydantic import ValidationError
 
 from verifiers.v1 import graph
 from verifiers.v1.clients.config import ClientConfig, TrainClientConfig
+from verifiers.v1.configs.agent import AgentConfig
 from verifiers.v1.episode import Episode
 from verifiers.v1.serve.server import EnvServer
 from verifiers.v1.serve.types import (
@@ -34,6 +35,7 @@ from verifiers.v1.serve.types import (
 )
 from verifiers.v1.task import WireTaskData
 from verifiers.v1.trace import (
+    AgentInfo,
     Error,
     GenerationSpan,
     ModelCall,
@@ -227,7 +229,6 @@ def _timing(raw: Any) -> Timing:
 _V0_TO_V1_TRUNCATION_STOP = {
     "max_turns_reached": "max_turns",
     "prompt_too_long": "context_length",
-    "timeout_reached": "harness_timeout",
     "max_total_completion_tokens_reached": "max_output_tokens",
 }
 
@@ -270,7 +271,10 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             type="Task",
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
-        tools=_to_v1_tools(out.get("tool_defs")),
+        # v0 rollouts carry no agent config — record the default, like the
+        # base task type above.
+        agent=AgentInfo(config=AgentConfig()),
+        tools=_to_v1_tools(out.get("tool_defs")) or [],
         rewards={"reward": Reward(score=float(out.get("reward") or 0.0))},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         info=dict(out.get("info") or {}),

verifiers/v1/push.py

@@ -58,8 +58,8 @@ def trace_to_sample(
         "example_id": trace.task.data.idx,
         "rollout_number": rollout_number,
         "episode_id": episode_id,
-        "agent": trace.agent_name,
-        "trainable": trace.trainable,
+        "agent": trace.agent.name,
+        "trainable": trace.agent.trainable,
         "task": task,
         "prompt": [],
         "completion": dump(branches[-1].messages) if branches else [],
@@ -73,8 +73,8 @@ def trace_to_sample(
         "is_completed": trace.is_completed,
         "is_truncated": trace.is_truncated,
         "metrics": trace.metrics,
-        "error": trace.error.model_dump(mode="json", exclude_none=True)
-        if trace.error
+        "error": trace.last_error.model_dump(mode="json", exclude_none=True)
+        if trace.last_error
         else None,
         "stop_condition": trace.stop_condition,
         "trajectory": [
@@ -124,7 +124,7 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     back to all traces when none are trainable (same rule as the dashboard).
     `avg_error` is the share of EPISODES that aren't ok: a hook failure counts
     even when its traces are clean or it left none."""
-    scored = [t for t in traces if t.trainable] or traces
+    scored = [t for t in traces if t.agent.trainable] or traces
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:

verifiers/v1/retries.py

@@ -117,7 +117,9 @@ async def run_episode_with_retry(
         final = await run()
         if attempt == retry.max_retries or not episode_should_retry(final, retry):
             break
-        cause = final.error or next((t.error for t in final.traces if t.error), None)
+        cause = final.error or next(
+            (t.last_error for t in final.traces if t.last_error), None
+        )
         history.extend(final.errors)
         for trace in final.traces:
             history.extend(trace.errors)