verifiers 0.3.0__tar.gz → 0.3.1.dev5__tar.gz

{verifiers-0.3.0 → verifiers-0.3.1.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.0
+Version: 0.3.1.dev5
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.0 → verifiers-0.3.1.dev5}/verifiers/v1/episode.py

@@ -1,14 +1,14 @@
 """The episode — one run's traces plus their shared standing, whole."""
 
 import uuid
-from typing import Generic
+from typing import Any, Generic
 
 from pydantic import BaseModel, Field
 
 from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import DataT, WireTaskData
-from verifiers.v1.trace import AgentConfigT, Error, Trace
+from verifiers.v1.trace import EXCLUDE_FIELDS, AgentConfigT, Error, Trace
 from verifiers.v1.types import Usage
 
 
@@ -72,6 +72,14 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
             grouped.setdefault(trace.agent.name, []).append(trace)
         return grouped
 
+    def to_record(self) -> dict[str, Any]:
+        """JSON record without raw trace tensors, which remain on the msgpack wire."""
+        return self.model_dump(
+            mode="json",
+            exclude={"traces": {"__all__": EXCLUDE_FIELDS}},
+            exclude_none=True,
+        )
+
     @classmethod
     def of(cls, trace: Trace, env: str = "") -> "Episode":
         """The single-agent record: one trace as its own episode."""

{verifiers-0.3.0 → verifiers-0.3.1.dev5}/verifiers/v1/trace.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import time
 import traceback
 import uuid
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Iterable, Iterator, Mapping
 from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal
 
 import numpy as np
@@ -171,6 +171,20 @@ class ModelCall(BaseModel):
     """The failure that ended this call, coupled to the exchange that caused it."""
 
 
+def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall, int]]:
+    """Per-call lower bound on newly fed-in tokens: each call's prompt beyond the
+    previous call's final sequence, clamped at zero. Exact while the history is
+    append-only; tokens the engine drops between calls (stripped reasoning, truncated
+    history) register as an undercount of that call's new input rather than going
+    negative."""
+    prev_total = 0
+    for call in calls:
+        if call.usage is None:
+            continue
+        yield call, max(0, call.usage.input_tokens - prev_total)
+        prev_total = call.usage.total_tokens
+
+
 class Branch(BaseModel):
     """A root-to-leaf graph path; each branch becomes one training sample."""
 
@@ -311,7 +325,9 @@ class Branch(BaseModel):
 
     @property
     def num_input_tokens(self) -> int:
-        return self.num_total_tokens - self.num_output_tokens
+        """Fed-in tokens (system + user + tool), counted once; a lower bound whenever
+        the engine drops tokens between calls."""
+        return sum(increment for _, increment in min_new_input_tokens(self.calls))
 
 
 class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
@@ -372,8 +388,18 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     @property
     def num_input_tokens(self) -> int:
-        """Fed-in tokens (system + user + tool), counted once, summed across branches."""
-        return sum(branch.num_input_tokens for branch in self.branches)
+        """Fed-in tokens (system + user + tool), counted once across all branches —
+        calls on shared branch prefixes contribute once, not once per branch."""
+        seen: set[int] = set()
+        total = 0
+        for branch in self.branches:
+            # A call's predecessors are fixed by the graph, so its increment is the
+            # same in every branch containing it; keep the first occurrence.
+            for call, increment in min_new_input_tokens(branch.calls):
+                if id(call) not in seen:
+                    seen.add(id(call))
+                    total += increment
+        return total
 
     @property
     def num_output_tokens(self) -> int:

{verifiers-0.3.0 → verifiers-0.3.1.dev5}/verifiers/v1/utils/platform.py

@@ -1,8 +1,8 @@
 """Push a finished eval run to the Prime Intellect platform (`--no-push` to skip).
 
-Converts each v1 `Trace` to the platform's (v0) eval-sample schema and uploads the
-run over the `/evaluations/` API (create -> push samples -> finalize) — the same
-contract as `prime eval push`, done inline at the end of a run. Auth + base URL
+Uploads one sample per v1 `Episode` over the `/evaluations/` API (create -> push
+samples -> finalize). Each sample keeps the complete native Episode as its source
+of truth and includes a flat summary for older Platform consumers. Auth + base URL
 come from `$PRIME_API_KEY` / `~/.prime/config.json`.
 """
 
@@ -27,6 +27,17 @@ DEFAULT_FRONTEND_URL = "https://app.primeintellect.ai"
 _MAX_SAMPLES_PAYLOAD_BYTES = 25 * 1024 * 1024
 
 
+def json_bytes(value: Any) -> int:
+    return len(
+        json.dumps(
+            value,
+            ensure_ascii=False,
+            separators=(",", ":"),
+            allow_nan=False,
+        ).encode("utf-8")
+    )
+
+
 @dataclass
 class PushState:
     """Mutable upload status shared with the dashboard."""
@@ -98,7 +109,7 @@ def trace_to_sample(
     return sample
 
 
-def _creds() -> tuple[str | None, str, str, str | None]:
+def credentials() -> tuple[str | None, str, str, str | None]:
     """(api_key, api_base, frontend_url, team_id) from env vars / `~/.prime/config.json`."""
     cfg = load_prime_config()
     api_key = os.getenv("PRIME_API_KEY") or cfg.get("api_key")
@@ -118,7 +129,7 @@ def _creds() -> tuple[str | None, str, str, str | None]:
     return api_key, base, frontend, team_id
 
 
-def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]:
+def run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]:
     """Run-level aggregates as v0's `GenerateMetadata`. Rewards/metrics aggregate
     over the trainable traces only — fixed agents (a judge, a modeled user) often
     carry no rewards and would dilute every mean with structural zeros — falling
@@ -149,23 +160,47 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     }
 
 
-def _build_samples(episodes: list[Episode]) -> list[dict[str, Any]]:
-    """One platform sample per trace, with one `rollout_number` per EPISODE: a
-    multi-agent rollout's seats are the same attempt at the task, not attempts
-    1..n. The grouping is the episode envelope — every trace in one episode shares
-    its `rollout_number` and `episode_id`."""
+def build_samples(episodes: list[Episode]) -> list[dict[str, Any]]:
+    """One Platform sample per Episode, with a legacy-compatible trace summary.
+
+    The native Episode in `info.native_wrapper` is authoritative and contains every
+    trace. One trainable trace (or the first trace) supplies only the flat summary
+    used by older consumers. `native_trace_index` identifies that summary trace.
+    """
     counts: dict[int, int] = {}
-    episode_numbers: dict[str, int] = {}
     samples = []
     for episode in episodes:
-        # One rollout_number per episode: all its seats are the same attempt.
-        number = episode_numbers.get(episode.id)
-        for trace in episode.traces:
-            if number is None:
-                idx = trace.task.data.idx
-                counts[idx] = number = counts.get(idx, 0) + 1
-                episode_numbers[episode.id] = number
-            samples.append(trace_to_sample(trace, number, episode.id))
+        if not episode.traces:
+            continue
+        summary_trace_index = next(
+            (
+                index
+                for index, candidate in enumerate(episode.traces)
+                if candidate.agent.trainable
+            ),
+            0,
+        )
+        summary_trace = episode.traces[summary_trace_index]
+        idx = summary_trace.task.data.idx
+        counts[idx] = number = counts.get(idx, 0) + 1
+        sample = trace_to_sample(summary_trace, number, episode.id)
+        sample["sample_id"] = episode.id
+        sample["info"] = {
+            **(sample["info"] or {}),
+            "native_wrapper": episode.to_record(),
+            "native_trace_index": summary_trace_index,
+        }
+        if len(b'{"samples":[]}') + json_bytes(sample) <= _MAX_SAMPLES_PAYLOAD_BYTES:
+            samples.append(sample)
+            continue
+        logger.warning(
+            "Episode %s exceeds the Platform sample limit; uploading projected traces",
+            episode.id,
+        )
+        samples.extend(
+            trace_to_sample(candidate, number, episode.id)
+            for candidate in episode.traces
+        )
     return samples
 
 
@@ -186,7 +221,7 @@ def push_traces(
             state.done = True
         return url
 
-    api_key, base, frontend, team_id = _creds()
+    api_key, base, frontend, team_id = credentials()
     if not api_key:
         logger.warning(
             "--push: no PRIME_API_KEY (set it or run `prime login`); skipping upload"
@@ -195,8 +230,7 @@ def push_traces(
 
     traces = [trace for episode in episodes for trace in episode.traces]
     env_name = (config.env.taskset.id) or config.legacy.id
-    metrics = _run_metrics(episodes, traces)
-    samples = _build_samples(episodes)
+    metrics = run_metrics(episodes, traces)
     num_examples = len({t.task.data.idx for t in traces})
     metadata = {
         "framework": "verifiers",
@@ -213,18 +247,12 @@ def push_traces(
     # The run is done and its results saved; a network blip here must not crash it
     # — log and skip the upload instead.
     try:
+        samples = build_samples(episodes)
         batches: list[list[dict[str, Any]]] = []
         batch: list[dict[str, Any]] = []
         payload_bytes = len(b'{"samples":[]}')
         for i, sample in enumerate(samples):
-            sample_bytes = len(
-                json.dumps(
-                    sample,
-                    ensure_ascii=False,
-                    separators=(",", ":"),
-                    allow_nan=False,
-                ).encode("utf-8")
-            )
+            sample_bytes = json_bytes(sample)
             sample_payload_bytes = len(b'{"samples":[]}') + sample_bytes
             if sample_payload_bytes > _MAX_SAMPLES_PAYLOAD_BYTES:
                 raise ValueError(