verifiers 0.3.2.dev137__py3-none-any.whl → 0.3.2.dev139__py3-none-any.whl

verifiers/v1/agent.py

@@ -12,13 +12,15 @@ import logging
 from collections.abc import AsyncIterator, Callable, Mapping
 from contextlib import asynccontextmanager, nullcontext
 from dataclasses import dataclass, replace
-from typing import Self
+from typing import Generic, Self, cast
+
+from typing_extensions import TypeVar
 
 from verifiers.v1.clients import (
     EvalClientConfig,
     ModelContext,
 )
-from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
+from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig, agent_config_fields
 from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects import parse_message
 from verifiers.v1.harness import Harness
@@ -42,6 +44,7 @@ from verifiers.v1.types import (
     ToolMessage,
     UserMessage,
 )
+from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.compile import (
     cap_remote_agent_timeout,
     resolve_runtime_config,
@@ -462,9 +465,8 @@ class Agent:
                     run.trace.stop("agent_completed")
             trace = await run.close()
         except BaseException:
-            # A cancellation mid-run (or a lifetime bug raised to the caller) means
-            # close() never runs — free the run's servers and owned runtime first.
-            await run.abort()
+            # Finish cleanup even if another cancellation arrives during abort().
+            await run_shielded(run.abort())
             raise
         if trace.agent.runtime is not None:
             trace.agent.runtime.borrowed = runtime is not None
@@ -720,27 +722,22 @@ def make_agent(
     return Agent(config, interception=interception)
 
 
-MakeAgent = Callable[[str, AgentConfig], Agent]
+AgentT = TypeVar("AgentT", bound=Agent, default=Agent)
+MakeAgent = Callable[[str, AgentConfig], AgentT]
 """An agent factory keyed by name — what `Agents` calls per scraped config field."""
 
 
-def agent_config_fields(config) -> dict[str, AgentConfig]:
-    """The top-level `AgentConfig` fields declared on a config, in declaration
-    order — the env's agents, keyed by field name (the only naming site)."""
-    return {name: value for name, value in config if isinstance(value, AgentConfig)}
-
-
-class Agents:
+class Agents(Generic[AgentT]):
     """A config's agents, addressed by attribute: every top-level `AgentConfig`
     field becomes an `Agent` under the field's name (`agents.solver`)."""
 
-    def __init__(self, config, make: MakeAgent | None = None) -> None:
-        self._agents: dict[str, Agent] = {
-            name: make_agent(value) if make is None else make(name, value)
+    def __init__(self, config, make: MakeAgent[AgentT] | None = None) -> None:
+        self._agents: dict[str, AgentT] = {
+            name: cast(AgentT, make_agent(value)) if make is None else make(name, value)
             for name, value in agent_config_fields(config).items()
         }
 
-    def __getattr__(self, name: str) -> Agent:
+    def __getattr__(self, name: str) -> AgentT:
         # self.__dict__ directly: attribute lookup re-entering __getattr__ before
         # __init__ ran (copy/unpickle) must raise, not recurse.
         agents = self.__dict__.get("_agents")

verifiers/v1/cli/dashboard/eval.py

@@ -81,9 +81,9 @@ def format_cost_usd(cost_usd: float) -> str:
 def _seat_value(config: EvalConfig, read):
     """A cap as the overview shows it: the declared seats' shared value, the
     string 'per-seat' when they disagree (caps live on the seats)."""
-    from verifiers.v1.configs.env import _declared_agent_configs
+    from verifiers.v1.configs.agent import agent_config_fields
 
-    values = {read(spec) for spec in _declared_agent_configs(config.env).values()}
+    values = {read(spec) for spec in agent_config_fields(config.env).values()}
     return values.pop() if len(values) == 1 else "per-seat"
 
 

verifiers/v1/cli/debug.py

@@ -17,7 +17,7 @@ from uuid import uuid4
 from pydantic_config import cli
 
 import verifiers.v1 as vf
-from verifiers.v1.cli.output import append_trace, save_config
+from verifiers.v1.cli.output import save_config
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_taskset_config,
@@ -34,6 +34,7 @@ from verifiers.v1.utils.compile import resolve_runtime_config
 from verifiers.v1.utils.decorators import invoke
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.trace_store import append_trace
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/cli/eval/main.py

@@ -10,7 +10,6 @@ from pydantic_config import cli
 
 from verifiers.v1.cli.eval.runner import run_eval
 from verifiers.v1.cli.output import (
-    TRACES_FILE,
     create_attempt_log_dir,
     output_path,
     saved_config_path,
@@ -26,6 +25,7 @@ from verifiers.v1.cli.resolve import (
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.trace_store import TRACES_FILE
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/cli/eval/resume.py

@@ -16,9 +16,9 @@ from pathlib import Path
 
 from pydantic_core import from_json
 
-from verifiers.v1.cli.output import TRACES_FILE
 from verifiers.v1.episode import WireEpisode
 from verifiers.v1.task import task_key
+from verifiers.v1.utils.trace_store import TRACES_FILE
 
 
 def load(

verifiers/v1/cli/eval/runner.py

@@ -17,11 +17,7 @@ from typing import TypeVar, cast
 from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
 from verifiers.v1.cli.eval.hint import PRIME_RL_HINT
-from verifiers.v1.cli.output import (
-    append_episode,
-    output_path,
-    save_config,
-)
+from verifiers.v1.cli.output import output_path, save_config
 from verifiers.v1.cli.resume import distribute
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
@@ -35,6 +31,7 @@ from verifiers.v1.utils.platform import (
     log_episodes,
     open_run,
 )
+from verifiers.v1.utils.trace_store import append_episode
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/cli/gepa.py

@@ -16,7 +16,7 @@ import uuid
 from pydantic_config import cli
 
 import verifiers.v1 as vf
-from verifiers.v1.cli.output import TRACES_FILE, output_path, write_config
+from verifiers.v1.cli.output import output_path, write_config
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_config,
@@ -27,6 +27,7 @@ from verifiers.v1.cli.resolve import (
 from verifiers.v1.gepa import GEPAConfig, run_gepa
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.trace_store import TRACES_FILE
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/cli/output.py

@@ -1,31 +1,13 @@
-"""On-disk output: traces.jsonl (one rollout episode per line) + configs/<cli>.json.
-
-Each line is an `Episode` — the episode's standing (`id`/`env`/`errors`) inlined
-next to its flat, self-contained traces — so an episode persists whole or not at all: a torn line is the
-whole episode owed on resume, and a failure before any trace minted still leaves
-its errors on disk. The JSON file is the run's resolved config in the format the
-CLI reads (`@ configs/<cli>.json`), so a run is re-runnable from its own output. Lines
-append as episodes complete, so results are durable mid-run. Files written
-by this surface contain episodes only.
-"""
-
-import asyncio
+"""CLI output directories, saved launch/config files, and per-attempt log paths."""
+
 import json
 import os
-from functools import cache
 from pathlib import Path
 
-from pydantic import BaseModel, TypeAdapter
+from pydantic import BaseModel
 
 from verifiers.v1.configs.cli.eval import EvalConfig
-from verifiers.v1.episode import EnvInfo, Episode, WireEpisode
-from verifiers.v1.state import StateT
-from verifiers.v1.task import DataT
-from verifiers.v1.trace import AgentConfigT, Trace
-from verifiers.v1.utils.aio import run_shielded
-
-TRACES_FILE = "traces.jsonl"
-"""Filename a run's rollout episodes are written to (one JSON episode per line)."""
+from verifiers.v1.utils.trace_store import TRACES_FILE
 
 CONFIG_DIR = "configs"
 """Directory inside a run dir holding its configs: the launch TOML copied verbatim to
@@ -78,10 +60,6 @@ def saved_config_path(run_dir: Path) -> Path | None:
     return candidates[0] if candidates else None
 
 
-# Compiling an adapter is the expensive part; run output reuses only a few model classes.
-type_adapter = cache(TypeAdapter)
-
-
 def output_path(config: EvalConfig) -> Path:
     """Where this run writes: `output_dir / run.dir` — the same grouping convention as
     training. The run directory defaults to the auto-generated run name
@@ -142,64 +120,3 @@ def save_config(
     (results_dir / TRACES_FILE).write_text(
         ""
     )  # fresh; appended to as rollouts complete
-
-
-def write_episode(
-    results_dir: Path, episode: Episode[DataT, StateT, AgentConfigT]
-) -> None:
-    """Serialize and append one rollout episode in the worker thread."""
-    # Preserve fields declared by typed Trace subclasses nested in the episode.
-    data = type_adapter(type(episode)).dump_json(episode, exclude_none=True)
-    with (results_dir / TRACES_FILE).open("ab") as f:
-        f.write(data + b"\n")
-
-
-def read_episodes(results_dir: Path, trace_type: type) -> list[WireEpisode]:
-    """Load a run's saved rollouts from `traces.jsonl` with traces typed as
-    `trace_type` (`Trace[WireTaskData, ...]` reads any taskset's file without
-    importing it)."""
-    trace_adapter = type_adapter(trace_type)
-    episodes: list[WireEpisode] = []
-    with (results_dir / TRACES_FILE).open(encoding="utf-8") as f:
-        for line in f:
-            if not line.strip():
-                continue
-            row = json.loads(line)
-            record = WireEpisode.model_validate({**row, "traces": []})
-            record.traces = [
-                trace_adapter.validate_python(trace) for trace in row["traces"]
-            ]
-            episodes.append(record)
-    return episodes
-
-
-async def append_episode(
-    results_dir: Path,
-    episode: Episode[DataT, StateT, AgentConfigT],
-    lock: asyncio.Lock,
-) -> None:
-    """Append one finished rollout episode without blocking the event loop. The run's
-    shared lock preserves whole-line ordering, and awaiting the worker preserves
-    per-episode durability."""
-
-    async def persist() -> None:
-        async with lock:
-            await asyncio.to_thread(write_episode, results_dir, episode)
-
-    # Run lock acquisition and the worker to completion even under cancellation, so
-    # finalized episodes are never lost mid-write (`run_shielded` re-raises the cancellation).
-    await run_shielded(persist())
-
-
-async def append_trace(
-    results_dir: Path, trace: Trace, lock: asyncio.Lock, env: str = ""
-) -> None:
-    """Append one finished trace as a single-agent rollout episode — debug and replay,
-    which complete trace-at-a-time, both go through here."""
-    episode = Episode(
-        env=EnvInfo(id=env),
-        task=trace.task,
-        traces=[trace],
-        ok=trace.ok,
-    )
-    await append_episode(results_dir, episode, lock)

verifiers/v1/cli/replay.py

@@ -22,13 +22,7 @@ from pydantic_config import cli
 
 import verifiers.v1 as vf
 from verifiers.v1.cli.dashboard.replay import ReplayProgress, replay_dashboard
-from verifiers.v1.cli.output import (
-    append_trace,
-    read_episodes,
-    save_config,
-    saved_config_path,
-    write_config,
-)
+from verifiers.v1.cli.output import save_config, saved_config_path, write_config
 from verifiers.v1.cli.resolve import narrow_taskset_config
 from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.configs.cli.replay import ReplayConfig
@@ -37,6 +31,7 @@ from verifiers.v1.task import Task, WireTaskData
 from verifiers.v1.trace import Trace
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
+from verifiers.v1.utils.trace_store import append_trace, read_episodes
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/configs/agent.py

@@ -1,6 +1,8 @@
 """One env agent's config: who plays the seat, and its per-run caps."""
 
-from pydantic import SerializeAsAny, model_validator
+from typing import Any
+
+from pydantic import BaseModel, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.clients import ClientConfig
@@ -8,6 +10,7 @@ from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 from verifiers.v1.types import SamplingConfig
+from verifiers.v1.utils.generic import deep_merge
 
 
 class TimeoutConfig(BaseConfig):
@@ -78,3 +81,50 @@ class WireAgentConfig(AgentConfig):
     def _resolve_harness(cls, data):
         """Override: a record read resolves no plugins."""
         return data
+
+
+def agent_config_fields(config: BaseModel) -> dict[str, AgentConfig]:
+    """Top-level agent configs, in declaration order, keyed by their field names."""
+    return {name: value for name, value in config if isinstance(value, AgentConfig)}
+
+
+def merge_agent_defaults(config: type[BaseModel], data: Any) -> Any:
+    """Merge partial agent overrides onto their declared defaults."""
+    if isinstance(data, dict):
+        for name, field in config.model_fields.items():
+            if isinstance(field.default, AgentConfig) and isinstance(
+                data.get(name), dict
+            ):
+                data[name] = deep_merge(
+                    field.default.model_dump(exclude_none=True), data[name]
+                )
+    return data
+
+
+def resolve_agent(
+    spec: AgentConfig,
+    *,
+    model: str | None = None,
+    client: ClientConfig | None = None,
+    sampling: SamplingConfig | None = None,
+    harness: HarnessConfig | None = None,
+) -> AgentConfig:
+    """`spec` with what it leaves unset filled from the run's defaults; its own
+    sampling values merge over the run's. The one place a seat's identity resolves,
+    for configured agent roles."""
+    merged = spec.sampling if sampling is None else sampling
+    if sampling is not None and spec.sampling is not None:
+        merged = sampling.model_copy(
+            update=deep_merge(
+                sampling.model_dump(exclude_unset=True),
+                spec.sampling.model_dump(exclude_unset=True),
+            )
+        )
+    return spec.model_copy(
+        update={
+            "harness": spec.harness if spec.harness is not None else harness,
+            "model": spec.model if spec.model is not None else model,
+            "client": spec.client if spec.client is not None else client,
+            "sampling": merged,
+        }
+    )

verifiers/v1/configs/env.py

@@ -6,13 +6,16 @@ from typing import get_args
 from pydantic import Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
-from verifiers.v1.configs.agent import AgentConfig
+from verifiers.v1.configs.agent import (
+    AgentConfig,
+    agent_config_fields,
+    merge_agent_defaults,
+)
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.interception import ElasticInterceptionPoolConfig, InterceptionConfig
 from verifiers.v1.types import ID
-from verifiers.v1.utils.generic import deep_merge
 
 
 class TimeoutConfig(BaseConfig):
@@ -73,7 +76,7 @@ class EnvConfig(BaseConfig):
         default = default_agent_harness(self.taskset.id)
         return {
             name: cfg.harness if cfg.harness is not None else default
-            for name, cfg in _declared_agent_configs(self).items()
+            for name, cfg in agent_config_fields(self).items()
         }
 
     @model_validator(mode="before")
@@ -110,17 +113,7 @@ class EnvConfig(BaseConfig):
     @model_validator(mode="before")
     @classmethod
     def _merge_role_defaults(cls, data):
-        """Deep-merge partial role data over the field's declared default — plain
-        validation would replace the instance wholesale, resetting its other pins."""
-        if isinstance(data, dict):
-            for name, field in cls.model_fields.items():
-                if isinstance(field.default, AgentConfig) and isinstance(
-                    data.get(name), dict
-                ):
-                    data[name] = deep_merge(
-                        field.default.model_dump(exclude_none=True), data[name]
-                    )
-        return data
+        return merge_agent_defaults(cls, data)
 
     @classmethod
     def __pydantic_init_subclass__(cls, **kwargs):
@@ -146,16 +139,6 @@ class EnvConfig(BaseConfig):
                 )
 
 
-def _declared_agent_configs(config: EnvConfig) -> dict[str, AgentConfig]:
-    """The `AgentConfig` fields declared on an env's config, in declaration order —
-    the env's roles, each seat keyed by its field name (the only naming site)."""
-    return {
-        name: getattr(config, name)
-        for name, field in type(config).model_fields.items()
-        if isinstance(field.default, AgentConfig)
-    }
-
-
 def default_agent_harness(taskset_id: str) -> HarnessConfig:
     """What an unpinned role's `harness=None` resolves to: the taskset's bundled
     harness when it ships one, else the built-in `bash`."""

verifiers/v1/env.py

@@ -14,12 +14,12 @@ from typing import (
 
 from verifiers.v1.agent import Agent, Agents, _EpisodeAgent
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.configs.agent import AgentConfig
-from verifiers.v1.configs.env import (
-    EnvConfig,
-    _declared_agent_configs,
-    default_agent_harness,
+from verifiers.v1.configs.agent import (
+    AgentConfig,
+    agent_config_fields,
+    resolve_agent,
 )
+from verifiers.v1.configs.env import EnvConfig, default_agent_harness
 from verifiers.v1.episode import EnvInfo, Episode
 from verifiers.v1.errors import EnvError, boundary
 from verifiers.v1.harness import Harness, HarnessConfig
@@ -32,7 +32,7 @@ from verifiers.v1.mcp import SharedToolServer, serve_shared
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace, TraceTask
-from verifiers.v1.utils.generic import concrete_type, deep_merge
+from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
 from verifiers.v1.utils.retries import run_episode_with_retry
 
@@ -105,7 +105,7 @@ class Env(ABC, Generic[ConfigT]):
         self._default_harness = default_agent_harness(config.taskset.id)
         task_cls = type(self.taskset).task_type()
         self._task_cls: type[Task] = task_cls
-        self._agent_specs: dict[str, AgentConfig] = _declared_agent_configs(self.config)
+        self._agent_specs: dict[str, AgentConfig] = agent_config_fields(self.config)
         if not self._agent_specs:
             raise ValueError(
                 f"{type(self).__name__} declares no agents; declare each as an "
@@ -205,23 +205,12 @@ class Env(ABC, Generic[ConfigT]):
 
         def make(name: str, spec: AgentConfig) -> Agent:
             # Unpinned fields fall back to the run's ctx / the taskset's harness.
-            sampling = ctx.sampling
-            if spec.sampling is not None:
-                sampling = sampling.model_copy(
-                    update=deep_merge(
-                        sampling.model_dump(exclude_unset=True),
-                        spec.sampling.model_dump(exclude_unset=True),
-                    )
-                )
-            resolved = spec.model_copy(
-                update={
-                    "harness": spec.harness
-                    if spec.harness is not None
-                    else self._default_harness,
-                    "model": spec.model if spec.model is not None else ctx.model,
-                    "sampling": sampling,
-                    "client": spec.client if spec.client is not None else ctx.client,
-                }
+            resolved = resolve_agent(
+                spec,
+                model=ctx.model,
+                client=ctx.client,
+                sampling=ctx.sampling,
+                harness=self._default_harness,
             )
             return _EpisodeAgent(
                 resolved,

verifiers/v1/gepa/runner.py

@@ -11,7 +11,7 @@ import logging
 from gepa.api import optimize
 from gepa.core.result import GEPAResult
 
-from verifiers.v1.cli.output import append_episode, output_path, save_config
+from verifiers.v1.cli.output import output_path, save_config
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.env import Env
 from verifiers.v1.episode import Episode
@@ -22,6 +22,7 @@ from verifiers.v1.gepa.dataset import (
     split_tasks,
 )
 from verifiers.v1.gepa.reflection import build_reflection_lm
+from verifiers.v1.utils.trace_store import append_episode
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/runtimes/base.py

@@ -72,6 +72,10 @@ class RuntimeProcess(ABC):
     async def wait(self) -> int:
         pass
 
+    @abstractmethod
+    async def poll(self) -> int | None:
+        """Return the underlying process's exit code without waiting for it to exit."""
+
     @abstractmethod
     async def terminate(self) -> None:
         pass
@@ -134,7 +138,7 @@ class BaseRuntimeInfo(BaseConfig):
 class Runtime(ABC):
     __slots__ = ("env",)
 
-    is_local: ClassVar[bool] = True
+    is_local: bool = True
     """Whether this runtime exchanges host-local URLs without a public tunnel. True for
     subprocess and the local container runtimes; remote runtimes override to False and
     use a host `Tunnel` inward plus `expose` outward."""