verifiers 0.2.2.dev90__tar.gz → 0.2.2.dev92__tar.gz

{verifiers-0.2.2.dev90 → verifiers-0.2.2.dev92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev90
+Version: 0.2.2.dev92
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev90 → verifiers-0.2.2.dev92}/verifiers/v1/cli/eval/runner.py

@@ -10,6 +10,7 @@ from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
 from verifiers.v1.cli.output import (
     append_episode,
+    append_trace,
     output_path,
     save_config,
 )
@@ -76,7 +77,8 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     write_lock = asyncio.Lock()
 
     async def on_complete(episode: Episode) -> None:
-        episode.record_run(EvalRunInfo(id=config.uuid))
+        for trace in episode.traces:
+            trace.record_run(EvalRunInfo(id=config.uuid))
         await append_episode(out, episode, write_lock)
 
     # Serving resources (shared tool servers, interception) come up once for the
@@ -256,10 +258,9 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 )
             records = []
             for trace in traces:
-                record = Episode.of(trace, env=config.env_id)
-                record.record_run(EvalRunInfo(id=config.uuid))
-                await append_episode(out, record, write_lock)
-                records.append(record)
+                trace.record_run(EvalRunInfo(id=config.uuid))
+                await append_trace(out, trace, write_lock, env=config.env_id)
+                records.append(Episode.of(trace))
             return records
 
         async def run_unit(payload: dict) -> list[Episode]:
@@ -270,7 +271,8 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     sampling=config.sampling,
                     **payload,
                 )
-            episode.record_run(EvalRunInfo(id=config.uuid))
+            for trace in episode.traces:
+                trace.record_run(EvalRunInfo(id=config.uuid))
             await append_episode(out, episode, write_lock)
             return [episode]
 

{verifiers-0.2.2.dev90 → verifiers-0.2.2.dev92}/verifiers/v1/episode.py

@@ -1,14 +1,14 @@
 """The episode — one run's traces plus their shared standing, whole."""
 
 import uuid
-from typing import Any, Generic
+from typing import Generic
 
 from pydantic import BaseModel, Field
 
 from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import DataT, WireTaskData
-from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace
+from verifiers.v1.trace import AgentConfigT, Error, Trace
 from verifiers.v1.types import Usage
 
 
@@ -26,19 +26,12 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     env: EnvInfo = Field(default_factory=EnvInfo)
     """The env that produced this episode."""
-    run: RunInfo | None = None
-    """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than
-    on each trace because the episode is what a consumer dispatches, and an episode that produced
-    no traces would otherwise have nowhere to say which run it was."""
     ok: bool = False
     """Whether the episode completed successfully."""
     errors: list[Error] = Field(default_factory=list)
     """Every error captured across attempts, oldest to newest."""
     traces: list[Trace[DataT, StateT, AgentConfigT]] = Field(default_factory=list)
     """Every agent's trace, in completion order."""
-    info: dict[str, Any] = Field(default_factory=dict)
-    """Scratch space for episode-level metadata, the counterpart to `Trace.info`. What describes
-    the whole episode belongs here rather than repeated on each of its traces."""
 
     @property
     def last_error(self) -> Error | None:
@@ -79,13 +72,6 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
             grouped.setdefault(trace.agent.name, []).append(trace)
         return grouped
 
-    def record_run(self, run: RunInfo | None = None, **info: Any) -> None:
-        """Record the run identity and any extra metadata about this episode. Both describe the
-        episode as a whole, so they are recorded once here rather than repeated on every trace."""
-        if run is not None:
-            self.run = run
-        self.info.update(info)
-
     @classmethod
     def of(cls, trace: Trace, env: str = "") -> "Episode":
         """The single-agent record: one trace as its own episode."""

{verifiers-0.2.2.dev90 → verifiers-0.2.2.dev92}/verifiers/v1/trace.py

@@ -321,6 +321,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     """Unique ID for this trace, auto-generated."""
     verifiers: VersionInfo = Field(default_factory=_current_build)
     """The verifiers version that produced this trace."""
+    run: RunInfo | None = None
+    """The run this trace belongs to (eval or train), consumer-stamped."""
+
     task: TraceTask[DataT]
     """The task data that seeded this trace."""
     agent: AgentInfo[AgentConfigT]
@@ -494,6 +497,12 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         if response.usage is not None:
             self.extra_usage.append(response.usage)
 
+    def record_run(self, run: RunInfo | None = None, **info: Any) -> None:
+        """Record the run identity (eval / train), and optional extra info."""
+        if run is not None:
+            self.run = run
+        self.info.update(info)
+
     def stop(self, condition: str) -> None:
         """Stop the trace, optionally with a stop condition."""
         self.is_completed = True