verifiers 0.3.2.dev100__py3-none-any.whl → 0.3.2.dev101__py3-none-any.whl

verifiers/v1/cli/dashboard/eval.py

@@ -16,6 +16,7 @@ from rich.table import Table
 from rich.text import Text
 
 from verifiers.v1.cli.dashboard.base import live_view
+from verifiers.v1.cli.eval.hint import PRIME_RL_HINT
 from verifiers.v1.cli.output import attempt_log_file, output_path
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import RunSlot
@@ -280,6 +281,10 @@ def Overview(config: EvalConfig) -> Table:
     return grid
 
 
+def _prime_rl_footer() -> Group:
+    return Group(Text(""), Text(PRIME_RL_HINT, style="dim", overflow="fold"))
+
+
 def _push_footer(push: "PushState | None") -> Group | None:
     """The `--push` line under the rollouts: dim with the run's URL while it streams,
     white once pushed, red when it failed. `None` when `--push` is off or the run stayed
@@ -833,13 +838,15 @@ def _render(
     # The --push status line (and, on Ctrl-C, the cleanup notice) appear under the rollouts. Measure
     # the fixed top (header + progress + rule) and the footer so the rollout rows fill what's left;
     # page through them (timer / arrows) when they'd overflow (else rich truncates).
-    footers = [f for f in (_push_footer(push), _interrupt_footer()) if f is not None]
-    footer = Group(*footers) if footers else None
+    footers = [
+        f
+        for f in (_push_footer(push), _interrupt_footer(), _prime_rl_footer())
+        if f is not None
+    ]
+    footer = Group(*footers)
     progress = Progress(slots, start, completed)
     top = Group(header, progress, Rule(style="dim"))
-    reserved = len(_CONSOLE.render_lines(top))
-    if footer is not None:
-        reserved += len(_CONSOLE.render_lines(footer))
+    reserved = len(_CONSOLE.render_lines(top)) + len(_CONSOLE.render_lines(footer))
     rows_per_page = max(1, _CONSOLE.size.height - reserved - 1)
     if tail is not None:  # --show-logs: the run's log stream in place of rollout rows
         parts = [
@@ -847,9 +854,8 @@ def _render(
             progress,
             Rule(style="dim"),
             tail.view(rows_per_page),
+            footer,
         ]
-        if footer is not None:
-            parts.append(footer)
         return Group(*parts)
     page_groups, index, count = _paginate(_groups(slots), rows_per_page, pager, now)
     if count > 1:
@@ -859,9 +865,8 @@ def _render(
         progress,
         Rule(style="dim"),
         Rows(page_groups, now, runtime_type, completed),
+        footer,
     ]
-    if footer is not None:
-        parts.append(footer)
     return Group(*parts)
 
 

verifiers/v1/cli/debug.py

@@ -36,7 +36,7 @@ from verifiers.v1.utils.logging import setup_logging
 logger = logging.getLogger(__name__)
 
 USAGE = (
-    "usage: uv run debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
+    "usage: uv run vf-debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
     "[--runtime.type subprocess] [options] [@ file.toml]\n"
     "       runs setup, then one command or uploaded host script, and saves traces"
 )

verifiers/v1/cli/eval/hint.py

@@ -0,0 +1,5 @@
+PRIME_RL_HINT = (
+    "vf-eval will be deprecated - move to prime-rl's `uv run eval`: it streams traces "
+    "live into a dashboard, evaluates several envs in one run and scales through env "
+    "servers. https://github.com/PrimeIntellect-ai/prime-rl"
+)

verifiers/v1/cli/eval/main.py

@@ -30,8 +30,8 @@ from verifiers.v1.utils.logging import setup_logging
 logger = logging.getLogger(__name__)
 
 USAGE = (
-    "usage: uv run eval [<taskset-id>] [--env.id <id>] [options] [@ file.toml]\n"
-    "       uv run eval @ <run-dir>/configs/resolved/eval.json --resume   (re-run the run's missing/errored rollouts)"
+    "usage: uv run vf-eval [<taskset-id>] [--env.id <id>] [options] [@ file.toml]\n"
+    "       uv run vf-eval @ <run-dir>/configs/resolved/eval.json --resume   (re-run the run's missing/errored rollouts)"
 )
 
 
@@ -48,7 +48,7 @@ def main(argv: list[str] | None = None) -> None:
         return
     # An env-block flag skips the usage gate so the typed parse renders its
     # did-you-mean instead of a bare usage line.
-    typed_axis = any(a.startswith(("--env.", "--serve.")) for a in argv)
+    typed_axis = any(a.startswith("--env.") for a in argv)
     if (
         not extract_id(argv, "env.taskset")
         and not references_config_file(argv)
@@ -108,7 +108,7 @@ def main(argv: list[str] | None = None) -> None:
             raise SystemExit(
                 f"--resume requires the exact config the run was started with - it "
                 f"differs in [{', '.join(changed)}]. Resumed rollouts would not be "
-                f"comparable; re-run with `uv run eval @ {saved_path} --resume`, or "
+                f"comparable; re-run with `uv run vf-eval @ {saved_path} --resume`, or "
                 "start a fresh run"
             )
     if config.dry_run:  # resolved + validated; write it to the output dir and exit
@@ -116,8 +116,7 @@ def main(argv: list[str] | None = None) -> None:
         logger.info("wrote config to %s", write_config(config, run_path))
         return
     # Always tee this attempt's logs to `logs/attempt_<n>/eval.log` (`logs/latest`
-    # points there) — in server mode (the default) the workers write there too, and
-    # `--rich.show-logs` tails it live.
+    # points there); `--rich.show-logs` tails it live.
     log_file = str(create_attempt_log_dir(run_path) / "eval.log")
     level = "DEBUG" if config.verbose else "INFO"
     setup_logging(level, log_file=log_file, console=config.rich is None)
@@ -128,7 +127,6 @@ def main(argv: list[str] | None = None) -> None:
     install_interrupt()
 
     try:
-        # Through the env-server worker pool by default; in-process with --no-serve.
         episodes = asyncio.run(run_eval(config))
     except KeyboardInterrupt:
         # Graceful cleanup has already run (each rollout's `finally`); partial results are on

verifiers/v1/cli/eval/runner.py

@@ -1,12 +1,9 @@
 """The eval runner: fan episodes out with bounded concurrency.
 
-Rollouts run through the env-server worker pool by default (`[serve]` sizes it;
-elastic — one worker, scaling on demand), the same path prime-rl trains through.
-`--no-serve` runs them in-process instead. Both paths share this runner — task
-selection, resume, persistence, the dashboard — and differ only in how one slot
-becomes one episode: `env.run_slot` in-process, a `run` request to the pool
-otherwise. The dashboard watches the same `RunSlot`s either way; a served slot
-has no live traces, so its per-turn detail lands when the episode completes.
+Rollouts run in-process: the env comes up once in this process and each slot becomes
+one episode through `env.run_slot`. The dashboard watches the same `RunSlot`s the env
+fills with live traces. Serving an env to many consumers is prime-rl's job (its `eval`
+runs env servers); this CLI is the quick local path.
 """
 
 import asyncio
@@ -18,16 +15,15 @@ from typing import TypeVar, cast
 
 from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
+from verifiers.v1.cli.eval.hint import PRIME_RL_HINT
 from verifiers.v1.cli.output import (
     append_episode,
-    attempt_log_file,
     output_path,
     save_config,
 )
 from verifiers.v1.cli.resume import distribute
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
-from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode, EvalRunInfo
 from verifiers.v1.utils.aio import run_shielded
@@ -84,93 +80,11 @@ async def _in_process(
         yield run
 
 
-@contextlib.asynccontextmanager
-async def _server(
-    config: EvalConfig,
-    serve: ServeConfig,
-    semaphore: asyncio.Semaphore | None,
-    on_complete: OnComplete,
-) -> AsyncIterator[RunSlotFn]:
-    """Run slots through a spawned env-server worker pool: each rollout is its own
-    `run` request, dispatched least-busy across workers. The workers own the env
-    (and its serving resources); this process owns the taskset and the results."""
-    import multiprocessing as mp
-    from functools import partial
-
-    from verifiers.v1.configs.serve import pool_serve_kwargs
-    from verifiers.v1.serve import EnvClient, env_config_data, serve_env
-    from verifiers.v1.utils.logging import setup_logging
-
-    # Spawned processes inherit no logging — hand them the main process's setup so
-    # their rollout logs land in the output dir. They share its stderr, so console
-    # output follows the main process's choice: off under the dashboard (worker log
-    # lines would print over the Live view and shift it), on otherwise.
-    level = "DEBUG" if config.verbose else "INFO"
-    log_file = str(attempt_log_file(output_path(config)))
-    console = config.rich is None
-    mpctx = mp.get_context("spawn")
-    address_queue: mp.Queue = mpctx.Queue()
-    # Death pipe: serve_env self-terminates if this process dies abruptly — we keep
-    # parent_conn, whose close (even on our SIGKILL) signals the child's watch.
-    parent_conn, child_conn = mpctx.Pipe()
-    proc = mpctx.Process(
-        target=serve_env,
-        kwargs=dict(
-            **pool_serve_kwargs(serve.pool),
-            address="tcp://127.0.0.1:0",
-            address_queue=address_queue,
-            death_pipe=child_conn,
-            log_setup=partial(setup_logging, level, log_file, console),
-            config_data=env_config_data(config.env),  # picklable across the spawn
-            # `-c` seeds each worker's episode bound unless `[serve]` pins one — so a
-            # pool carries `workers * bound` episodes, as `multiplex` implies.
-            max_concurrent=serve.max_concurrent
-            if serve.max_concurrent is not None
-            else config.max_concurrent,
-        ),
-        daemon=False,
-    )
-    proc.start()
-    child_conn.close()  # the child holds its end; we keep parent_conn so our exit closes it
-    try:
-        address = await asyncio.to_thread(address_queue.get, timeout=600)
-        client = EnvClient(address=address)
-        try:
-            await client.wait_for_server_startup(timeout=600)
-
-            async def run(slot: RunSlot) -> Episode:
-                async with semaphore or contextlib.nullcontext():
-                    slot.started = time.time()
-                    episode = await client.run(
-                        client=config.client,
-                        model=config.model,
-                        sampling=config.sampling,
-                        task_data=slot.task.data.model_dump(mode="json"),
-                    )
-                slot.traces = list(episode.traces)
-                slot.episode = cast(Episode, episode)
-                slot.done = True
-                await on_complete(cast(Episode, episode))
-                return cast(Episode, episode)
-
-            yield run
-        finally:
-            await client.close()
-    finally:
-        proc.terminate()
-        with contextlib.suppress(Exception):
-            await asyncio.to_thread(proc.join, 10)
-        with contextlib.suppress(Exception):
-            parent_conn.close()
-
-
 async def run_eval(config: EvalConfig) -> list[Episode]:
-    from verifiers.v1.utils.loaders import load_environment, load_taskset
+    from verifiers.v1.utils.loaders import load_environment
 
-    # The env comes up in this process only for an in-process run; a served run's
-    # workers each load their own, and this process owns just the taskset.
-    env = None if config.serve is not None else load_environment(config.env)
-    taskset = env.taskset if env is not None else load_taskset(config.env.taskset)
+    env = load_environment(config.env)
+    taskset = env.taskset
     if config.num_tasks is None and taskset.INFINITE:
         raise ValueError(
             f"{type(taskset).__name__} is infinite - bound the run with -n"
@@ -186,14 +100,13 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
     finished: list[Episode] = []
     if config.resume:
         keys = [task.hash for task in tasks]
-        # In-process, the env's own keep-verdict decides what resumes; a served run
-        # can't ask the worker-side env, so it keeps the default `episode.ok`.
-        complete = (
-            (lambda episode: env.complete(cast(Episode, episode)))
-            if env is not None
-            else None
+        # the env's own keep-verdict decides what resumes
+        loaded, owed = resume.load(
+            out,
+            keys,
+            config.num_rollouts,
+            lambda episode: env.complete(cast(Episode, episode)),
         )
-        loaded, owed = resume.load(out, keys, config.num_rollouts, complete)
         finished = [cast(Episode, episode) for episode in loaded]
         if not owed:  # already complete - report it and exit successfully
             print(
@@ -211,18 +124,10 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
         )
     else:
         save_config(config, out)
-        via = (
-            f" via the env-server {config.serve.pool.type} pool"
-            if config.serve is not None
-            else ""
-        )
         logger.info(
-            "running %dx%d rollouts on %s%s",
-            len(plan),
-            config.num_rollouts,
-            config.model,
-            via,
+            "running %dx%d rollouts on %s", len(plan), config.num_rollouts, config.model
         )
+        logger.info(PRIME_RL_HINT)
     start = time.time()
     logger.info("results: %s", out)
 
@@ -242,25 +147,11 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
         await append_episode(out, episode, write_lock)
         await asyncio.to_thread(log_episodes, run, [episode])
 
-    backend = (
-        _in_process(env, config, semaphore, on_complete)
-        if env is not None
-        else _server(config, config.serve, semaphore, on_complete)
-    )
-    # The run is closed out whatever breaks, backend setup and teardown included.
+    # The run is closed out whatever breaks, env setup and teardown included.
     try:
-        async with backend as run_slot:
-            # The display slots: in-process ones are the env's own (it fills their live
-            # traces); a served rollout's is a client-side stand-in its worker never sees.
-            planned = [
-                slot
-                for task, n in plan
-                for slot in (
-                    env.slots(task, n)
-                    if env is not None
-                    else [RunSlot(task) for _ in range(n)]
-                )
-            ]
+        async with _in_process(env, config, semaphore, on_complete) as run_slot:
+            # the env's own slots: it fills their live traces as the rollouts run
+            planned = [slot for task, n in plan for slot in env.slots(task, n)]
             slots = [RunSlot.finished(episode) for episode in finished] + planned
             display = (
                 dashboard(slots, config, start, push=push_state)

verifiers/v1/cli/gepa.py

@@ -1,4 +1,4 @@
-"""The GEPA entrypoint: `uv run gepa [<taskset-id>] --model <model> [options]`.
+"""The GEPA entrypoint: `uv run vf-gepa [<taskset-id>] --model <model> [options]`.
 
 Registered as the `gepa` console script. Optimizes a v1 taskset's `Task.system_prompt` via
 GEPA (Genetic-Pareto): alternating rollouts with a teacher LM reflecting on results — see
@@ -28,7 +28,7 @@ from verifiers.v1.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
 
-USAGE = "usage: uv run gepa [<taskset-id>] [--env.id <id>] --model <model> [options] [@ file.toml]"
+USAGE = "usage: uv run vf-gepa [<taskset-id>] [--env.id <id>] --model <model> [options] [@ file.toml]"
 
 
 def main(argv: list[str] | None = None) -> None:

verifiers/v1/cli/init.py

@@ -8,7 +8,7 @@ from pydantic_config import cli
 from verifiers.v1.configs.cli.init import InitConfig
 
 USAGE = (
-    "usage: uv run init <name> [--path ./environments] [-T/--add-tool] "
+    "usage: uv run vf-init <name> [--path ./environments] [-T/--add-tool] "
     "[-H/--add-harness]\n"
     "       scaffold a new environment package"
 )
@@ -191,7 +191,7 @@ A v1 verifiers environment, scaffolded with `init`.
 
 ```bash
 uv pip install -e .        # install this package (or register it in your project)
-uv run eval {dash} -n 3    # evaluate a few tasks with the bash harness
+uv run vf-eval {dash} -n 3    # evaluate a few tasks with the bash harness
 ```
 
 ## Layout
@@ -228,7 +228,7 @@ def scaffold(config: InitConfig) -> Path:
         _write(pkg_dir / "servers" / "__init__.py", "")
         _write(pkg_dir / "servers" / "tool.py", _tool_py(stem, prefix))
 
-    print(f"\ndone. next:\n  uv pip install -e {env_dir}\n  uv run eval {dash} -n 3")
+    print(f"\ndone. next:\n  uv pip install -e {env_dir}\n  uv run vf-eval {dash} -n 3")
     return env_dir
 
 

verifiers/v1/cli/replay.py

@@ -39,7 +39,7 @@ from verifiers.v1.utils.logging import setup_logging
 logger = logging.getLogger(__name__)
 
 USAGE = (
-    "usage: uv run replay <output-dir> [options] [@ file.toml]\n"
+    "usage: uv run vf-replay <output-dir> [options] [@ file.toml]\n"
     "       re-score a finished run's saved traces (judges + trace-only signals; no runtime)"
 )
 

verifiers/v1/cli/validate.py

@@ -49,9 +49,9 @@ REASONS = ("valid", "invalid", "unchecked", "error", "timeout")
 ResultRow = dict[str, Any]
 
 USAGE = (
-    "usage: uv run validate [<taskset-id>] [--only-setup | --only-gold] "
+    "usage: uv run vf-validate [<taskset-id>] [--only-setup | --only-gold] "
     "[-o <output-dir>] [--runtime.type subprocess] [options] [@ file.toml]\n"
-    "       uv run validate @ <run-dir>/configs/validate.json --resume   (re-run missing/errored/timed-out tasks)\n"
+    "       uv run vf-validate @ <run-dir>/configs/validate.json --resume   (re-run missing/errored/timed-out tasks)\n"
     "       runs persisted gold and setup-only checks per task (no model)"
 )
 

verifiers/v1/configs/cli/eval.py

@@ -9,7 +9,6 @@ from pydantic_config import BaseConfig
 from verifiers.v1.clients import ClientConfig, EvalClientConfig
 from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
 from verifiers.v1.configs.env import EnvConfig
-from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
 
@@ -76,10 +75,6 @@ class EvalConfig(BaseConfig):
     env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig()
     """The environment — which env, its seed taskset, each agent, its knobs. Narrowed to
     the selected env's config class by the env id, else the taskset id."""
-    serve: ServeConfig | None = Field(default_factory=ServeConfig)
-    """How the env is hosted: the env-server worker pool (elastic by default) and each
-    worker's episode bound — the path prime-rl trains through. `--no-serve` runs the
-    rollouts in-process instead."""
     run: RunConfig = Field(default_factory=RunConfig)
     """Run identity: `run.name` is the display name, `run.dir` names the directory
     under `output_dir`, and `run.id` is stamped on traces."""
@@ -110,8 +105,7 @@ class EvalConfig(BaseConfig):
     )
     """Episodes in flight at once, `None` for no limit. An episode plays its agents one
     at a time, so this is the live agent runs too — until `--env.max-concurrent-agents`
-    says otherwise. Under `[serve]` it also seeds each worker's bound, unless
-    `--serve.max-concurrent` pins one."""
+    says otherwise."""
     verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
     """Log at debug level instead of the default info."""
     dry_run: bool = Field(False, exclude=True)
@@ -122,8 +116,7 @@ class EvalConfig(BaseConfig):
     previous run's results. Excluded from the saved config."""
     rich: RichConfig | None = Field(default_factory=RichConfig)
     """The live dashboard (on by default; `--no-rich` streams logs to the console
-    instead). A served run has no live per-turn view, so its rollout rows fill in as
-    each episode completes; `--rich.show-logs` swaps the rows for the run's logs."""
+    instead); `--rich.show-logs` swaps the rollout rows for the run's logs."""
     push: bool = True
     """Upload the finished run to the Prime Intellect platform (the private Evaluations
     tab) at the end of the eval. On by default; disable with `--no-push`. Needs
@@ -136,7 +129,7 @@ class EvalConfig(BaseConfig):
     resume: bool = Field(False, exclude=True)
     """Re-run the run's missing/errored rollouts in place instead of starting fresh. The
     run dir comes from the resolved config (`output_dir / run.dir`), so resume with the
-    run's own config — e.g. `uv run eval @ <run-dir>/configs/eval.json --resume`. Excluded
+    run's own config — e.g. `uv run vf-eval @ <run-dir>/configs/eval.json --resume`. Excluded
     from the saved config."""
 
     @model_validator(mode="before")

verifiers/v1/configs/cli/validate.py

@@ -56,7 +56,7 @@ class ValidateConfig(BaseConfig):
     resume: bool = Field(False, exclude=True)
     """Re-run the run's missing, errored, and timed-out tasks in place. The run dir comes
     from the resolved config (`output_dir / run.dir`), so resume with the run's own
-    config — e.g. `uv run validate @ <run-dir>/configs/validate.json --resume`.
+    config — e.g. `uv run vf-validate @ <run-dir>/configs/validate.json --resume`.
     Excluded from the saved config."""
     clean: bool = Field(False, exclude=True)
     """Delete the run directory (`output_dir / run.dir`) before running, overwriting a

verifiers/v1/configs/client.py

@@ -4,7 +4,7 @@ A `BaseClientConfig` is an OpenAI-compatible endpoint (base_url + API-key env va
 + extra headers); `clients.resolve_client` turns one into a live `Client` — the
 interception server builds one per distinct config and shares it across the rollouts
 it multiplexes. The default Prime endpoint, API key, and team fall back to
-the active Prime CLI config, so direct `uv run eval` calls behave like `prime eval`.
+the active Prime CLI config, so direct `uv run vf-eval` calls behave like `prime eval`.
 Both the eval entrypoint (its model client) and in-env LLM calls (e.g. a judge reward)
 build clients from these. `ClientConfig` is the CLI-selectable discriminated union
 (eval | train).

verifiers/v1/env.py

@@ -316,17 +316,24 @@ class Env(ABC, Generic[ConfigT]):
         ctx: ModelContext,
         semaphore: asyncio.Semaphore | None = None,
         on_complete: Callable[[Episode], Awaitable[None]] | None = None,
+        on_trace: Callable[[Trace], None] | None = None,
     ) -> Episode:
         """Run one planned episode to completion, with whole-episode
         retries per `--env.retries`; `semaphore` bounds concurrent EPISODES — one
         permit for the attempt in flight, held across the whole of it (its agents,
         their boxes, `finalize()`) and released before a retry's backoff and before
-        `on_complete` (the runners' persistence hook, which fires when final)."""
+        `on_complete` (the runners' persistence hook, which fires when final).
+        `on_trace` sees each trace at mint, after it joined `slot.traces`."""
 
         async def attempt() -> Episode:
             slot.traces = []  # a retry shows the fresh attempt's traces
             live = slot.traces
 
+            def minted(trace: Trace) -> None:
+                live.append(trace)
+                if on_trace is not None:
+                    on_trace(trace)
+
             def discard(trace: Trace) -> None:
                 # A retried agent attempt abandons its trace; drop it from the view.
                 with contextlib.suppress(ValueError):
@@ -336,7 +343,7 @@ class Env(ABC, Generic[ConfigT]):
                 return await self.run_episode(
                     slot.task,
                     ctx,
-                    on_trace=live.append,
+                    on_trace=minted,
                     on_discard=discard,
                 )
 

verifiers/v1/graph.py

@@ -548,8 +548,13 @@ class PendingTurn:
         """Add this turn to the graph; returns the committed assistant node's id."""
         assistant_id = _commit_turn(self, response)
         self.trace.tools = self.tools
+        self.trace.clear_preview(self)
         return assistant_id
 
+    def abandon(self) -> None:
+        """The request failed or was cancelled before a commit: its preview goes."""
+        self.trace.clear_preview(self)
+
     def commit_prompt(self) -> None:
         """Record an input that terminated before model inference."""
         parent = self.prefix_node_ids[-1] if self.prefix_node_ids else None
@@ -570,6 +575,7 @@ class PendingTurn:
             parent = len(self.trace.nodes) - 1
             index[_node_key(previous, message, self.tools)] = parent
         self.trace.tools = self.tools
+        self.trace.clear_preview(self)
 
 
 def prepare_turn(

verifiers/v1/interception/server.py

@@ -483,6 +483,7 @@ class InterceptionServer(Interception):
                 acp=acp,
             )
         )
+        session.trace.notify()
 
     async def handle_request(
         self, request: web.Request, dialect: Dialect
@@ -660,6 +661,9 @@ class InterceptionServer(Interception):
             return web.json_response(dialect.error_body(str(error)), status=400)
         except RolloutError as error:
             return self._fail(session, dialect, error)
+        # The tail is what the harness added since the last turn (tool results, user
+        # turns): live watchers see it now rather than with the model's reply.
+        session.trace.preview(turn, turn.tail)
 
         inspect_response = bool(session.response_interceptors or session.response_stops)
         if relay_streaming:
@@ -782,6 +786,8 @@ class InterceptionServer(Interception):
                     error = e
                     raise
             finally:
+                if node is None:
+                    turn.abandon()
                 # The turn's one per-exchange record: settings, timing, outcome, and
                 # the error that ended it (if any).
                 self.record_call(
@@ -1047,6 +1053,8 @@ class InterceptionServer(Interception):
                 error = e
             raise
         finally:
+            if node is None:
+                turn.abandon()
             # The turn's one per-exchange record: settings, timing, outcome, and the
             # error that ended it (if any).
             self.record_call(

verifiers/v1/rollout.py

@@ -180,6 +180,7 @@ class Rollout:
         proceed; a setup failure is captured onto the trace."""
         self._opened = True
         self.trace.timing.boot.start = time.time()
+        self.trace.notify()
         if self._borrowed_runtime is None:
             self.runtime = make_runtime(self.runtime_config, name=self.trace.id)
         elif self._borrowed_runtime is not None and self._borrowed_runtime.stopped:
@@ -219,6 +220,7 @@ class Rollout:
             now = time.time()
             self.trace.timing.boot.end = now
             self.trace.timing.setup.start = now
+            self.trace.notify()
             # Task setup and harness provisioning share one setup-stage deadline.
             setup_deadline = (
                 None
@@ -342,6 +344,7 @@ class Rollout:
         now = time.time()
         self.trace.timing.setup.end = now
         self.trace.timing.agent.start = now
+        self.trace.notify()
         return not self._session.stopped
 
     async def step(self, messages: Messages | None = None) -> bool:
@@ -464,6 +467,7 @@ class Rollout:
             finally:
                 if trace.timing.agent.start and not trace.timing.agent.end:
                     trace.timing.agent.end = time.time()
+                trace.notify()
             if not self._failed and self._opened:
                 assert runtime is not None
                 trace.timing.finalize.start = time.time()
@@ -479,6 +483,7 @@ class Rollout:
                 now = time.time()
                 trace.timing.finalize.end = now
                 trace.timing.scoring.start = now
+                trace.notify()
                 async with boundary(TaskError, "scoring"):
                     # Cross-trace judgement runs later, after the runtime is gone.
                     await asyncio.wait_for(
@@ -489,6 +494,7 @@ class Rollout:
                         self._timeouts.scoring,
                     )
                 trace.timing.scoring.end = time.time()
+                trace.notify()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure
             self.fail(e)
         finally:

verifiers/v1/serve/__init__.py

@@ -1,4 +1,5 @@
 from verifiers.v1.serve.client import EnvClient
+from verifiers.v1.serve.delta import EpisodeAssembly
 from verifiers.v1.serve.pool import EnvServerPool, env_config_data, serve_env
 from verifiers.v1.serve.server import EnvServer
 from verifiers.v1.serve.types import (
@@ -12,6 +13,7 @@ __all__ = [
     "EnvClient",
     "EnvServer",
     "EnvServerPool",
+    "EpisodeAssembly",
     "HealthRequest",
     "HealthResponse",
     "RunRequest",