verifiers 0.2.1.dev55__tar.gz → 0.2.1.dev57__tar.gz

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev55
+Version: 0.2.1.dev57
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/__init__.py

@@ -100,12 +100,18 @@ from verifiers.v1.mcp import (
 )
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.trace import (
+    TRACE_VERSION,
+    AgentInfo,
     Branch,
     Error,
+    EvalRunInfo,
+    RunInfo,
     TimeSpan,
     Timing,
     Trace,
     TraceTask,
+    TrainRunInfo,
+    VersionInfo,
     WireTrace,
 )
 from verifiers.v1.types import (
@@ -162,6 +168,12 @@ __all__ = [
     "Trace",
     "TraceTask",
     "WireTrace",
+    "TRACE_VERSION",
+    "AgentInfo",
+    "RunInfo",
+    "EvalRunInfo",
+    "TrainRunInfo",
+    "VersionInfo",
     "State",
     "StateT",
     "MessageNode",

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/cli/eval/runner.py

@@ -12,7 +12,7 @@ from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.output import append_trace, output_path, save_config
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.env import Environment
-from verifiers.v1.trace import Trace
+from verifiers.v1.trace import EvalRunInfo, Trace
 from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
@@ -67,6 +67,7 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     write_lock = asyncio.Lock()
 
     async def on_complete(trace: Trace) -> None:
+        trace.stamp(EvalRunInfo(id=config.uuid))
         await append_trace(out, trace, write_lock)
 
     # Shared tool servers (if any) come up once here and their URLs flow into every rollout
@@ -221,6 +222,7 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
                     sampling=config.sampling,
                 )
             for trace in traces:
+                trace.stamp(EvalRunInfo(id=config.uuid))
                 await append_trace(out, trace, write_lock)
             return traces
 
@@ -232,6 +234,7 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
                     model=config.model,
                     sampling=config.sampling,
                 )
+            trace.stamp(EvalRunInfo(id=config.uuid))
             await append_trace(out, trace, write_lock)
             return [trace]
 

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/dialects/responses.py

@@ -21,10 +21,6 @@ from openai.types.responses import (
     ResponseUsage,
 )
 from openai.types.responses.response_input_param import FunctionCallOutput
-from openai.types.responses.response_usage import (
-    InputTokensDetails,
-    OutputTokensDetails,
-)
 from pydantic import BaseModel, ConfigDict
 
 from verifiers.v1.dialects.base import Dialect, StreamParser, iter_sse_reverse
@@ -58,11 +54,27 @@ _TERMINAL_MARKERS = tuple(
 _SAMPLING_KEYS = frozenset({"temperature", "top_p", "max_output_tokens", "max_tokens"})
 
 
+class ProviderUsageInputTokensDetails(BaseModel):
+    """Permissive input token details: OpenAI-compatible providers may omit fields
+    the pinned SDK declares required (e.g. ``cache_write_tokens``)."""
+
+    model_config = ConfigDict(extra="allow")
+    cache_write_tokens: int | None = None
+    cached_tokens: int | None = None
+
+
+class ProviderUsageOutputTokensDetails(BaseModel):
+    """Permissive output token details: providers may omit ``reasoning_tokens``."""
+
+    model_config = ConfigDict(extra="allow")
+    reasoning_tokens: int | None = None
+
+
 class ProviderUsage(ResponseUsage):
     """Responses usage with optional detail objects for OpenAI-compatible providers."""
 
-    input_tokens_details: InputTokensDetails | None = None
-    output_tokens_details: OutputTokensDetails | None = None
+    input_tokens_details: ProviderUsageInputTokensDetails | None = None
+    output_tokens_details: ProviderUsageOutputTokensDetails | None = None
 
 
 class OpenAIResponse(BaseModel):

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/graph.py

@@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from pydantic import ConfigDict, Field, field_serializer, field_validator
+from pydantic.json_schema import SkipJsonSchema
 from renderers.base import MultiModalData, PlaceholderRange, RenderedTokens
 
 from verifiers.v1.types import (
@@ -105,7 +106,7 @@ class MessageNode(StrictBaseModel):
     `mask`; empty for input messages."""
     finish_reason: FinishReason = None
     """The response's finish reason (assistant nodes only) — kept for truncation detection."""
-    multi_modal_data: MultiModalData | None = None
+    multi_modal_data: SkipJsonSchema[MultiModalData | None] = None
     """The renderer items for the images this message's content introduces (pixel tensors,
     grids, hashes, placeholders) — the only carrier of the pixels from the env server to the
     trainer. `Branch.multi_modal_data` concatenates them along the path into the training
@@ -114,13 +115,13 @@ class MessageNode(StrictBaseModel):
     usage: Usage | None = None
     """Provider-reported token usage for this message's response (assistant nodes). Preserved
     on the wire and on disk, including cache-read tokens when the provider reports them."""
-    routed_experts: np.ndarray | None = None
+    routed_experts: SkipJsonSchema[np.ndarray | None] = None
     """This node's slice of the MoE expert-routing array — uint8 `[len(token_ids), layers,
     top_k]`, the expert ids inference selected for exactly this node's tokens. Attributed from
     the turn's `generate` payload by `_attribute_routed_experts`; `Branch.routed_experts`
     concatenates these along the path into the trainer's router-replay input. Rides the wire as
     a raw-bytes `__nd__` dict; kept off disk by the dump-site `exclude` in prime-rl."""
-    kept_tokens: KeptTokens | None = None
+    kept_tokens: SkipJsonSchema[KeptTokens | None] = None
     """Kept-set sampling masks for this node's sampled tokens, decoded: `ids` flat int32
     in position order, `counts` the per-token kept-set sizes (aligned with `logprobs`;
     0 = no mask). Assistant nodes only; consumed via `Branch.kept_tokens` for

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/harness.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 import logging
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
-from typing import ClassVar, Generic, TypeVar
+from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 
 from pydantic import Field
 from pydantic_config import BaseConfig
@@ -18,9 +20,13 @@ from verifiers.v1.runtimes import (
     SubprocessConfig,
 )
 from verifiers.v1.task import TaskData
-from verifiers.v1.trace import Trace
 from verifiers.v1.types import ID, Messages
 
+if TYPE_CHECKING:
+    # Annotation-only: trace.py embeds HarnessConfig (AgentInfo), so the runtime
+    # import goes trace -> harness, not the other way around.
+    from verifiers.v1.trace import Trace
+
 logger = logging.getLogger(__name__)
 
 

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/rollout.py

@@ -5,6 +5,7 @@ from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from enum import StrEnum
 
+from verifiers import __version__
 from verifiers.v1.harness import Harness
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.decorators import discover_decorated, invoke
@@ -30,7 +31,8 @@ from verifiers.v1.runtimes import (
 from verifiers.v1.mcp import SharedToolServer, serve_tools, serve_user
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
-from verifiers.v1.trace import TraceTask, Trace
+from verifiers.v1.trace import AgentInfo, Trace, TraceTask, VersionInfo
+from verifiers.v1.utils.version import verifiers_commit
 
 logger = logging.getLogger(__name__)
 
@@ -109,6 +111,12 @@ class Rollout:
         trace: Trace = Trace(
             task=TraceTask(type=type(self.task).__name__, data=self.task.data),
             state=state_cls(type(self.task))(),
+            verifiers=VersionInfo(version=__version__, commit=verifiers_commit()),
+            agent=AgentInfo(
+                model=self.ctx.model,
+                sampling=self.ctx.sampling,
+                harness=self.harness.config,
+            ),
         )
         self.trace = trace  # expose for the --rich dashboard
         self.phase = Phase.BOOT  # leaving the queue: the runtime boots now

{verifiers-0.2.1.dev55 → verifiers-0.2.1.dev57}/verifiers/v1/trace.py

@@ -5,7 +5,7 @@ import time
 import traceback
 import uuid
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal, TypeVar
 
 import numpy as np
 from pydantic import Field, PrivateAttr
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
 from verifiers.v1 import graph
 from verifiers.v1.errors import ProviderError
 from verifiers.v1.graph import MessageNode
+from verifiers.v1.harness import HarnessConfig
 from verifiers.v1.runtimes import RuntimeInfo
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import DataT, WireTaskData
@@ -24,6 +25,7 @@ from verifiers.v1.types import (
     AssistantMessage,
     KeptTokens,
     Messages,
+    SamplingConfig,
     StrictBaseModel,
     Tool,
     ToolMessage,
@@ -212,6 +214,59 @@ _NODE_DUMP_EXCLUDE: dict = {
 """Raw tensor fields kept on the msgpack wire but excluded from JSON records."""
 
 
+TRACE_VERSION = 1
+"""Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on
+breaking shape changes; optional-with-default fields are additive and don't bump it."""
+
+
+class EvalRunInfo(StrictBaseModel):
+    """An eval run, stamped by the consumer (the eval CLI / a trainer's inline eval)."""
+
+    type: Literal["eval"] = "eval"
+    id: str | None = None
+    """The producing run: the eval CLI stamps its run uuid (a resumed eval counts as
+    a new run; kept traces keep their original id), trainers stamp their own."""
+    step: int | None = None
+    """The training step an inline eval was triggered at, stamped by the trainer;
+    None for a standalone eval (the eval CLI doesn't set it)."""
+
+
+class TrainRunInfo(StrictBaseModel):
+    """A training run, stamped by the trainer."""
+
+    type: Literal["train"] = "train"
+    id: str | None = None
+    """The trainer's run identifier."""
+    step: int | None = None
+    """The training step this rollout belongs to."""
+
+
+RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")]
+"""The run a trace belongs to, discriminated on `type`."""
+
+
+class VersionInfo(StrictBaseModel):
+    """The verifiers build that produced this trace."""
+
+    version: str
+    """The installed verifiers package version."""
+    commit: str | None = None
+    """The verifiers git commit, when resolvable (a git-pinned install or a source
+    checkout); None otherwise (e.g. a PyPI wheel)."""
+
+
+class AgentInfo(StrictBaseModel):
+    """The agent that produced this trace's sampled turns."""
+
+    model: str
+    """The model identifier requested from the client."""
+    sampling: SamplingConfig | None = None
+    """The resolved sampling settings the rollout ran with."""
+    harness: HarnessConfig | None = None
+    """The driving harness's config. Typed as the base config, so a custom harness's
+    extra fields don't serialize — records round-trip without importing the harness."""
+
+
 class TraceTask(StrictBaseModel, Generic[DataT]):
     """The task as recorded on the trace: the row (`data`, the wire half — fully typed,
     flows into scoring) plus the Task class name that produced the rollout (`type`) —
@@ -234,6 +289,15 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
     """The task being solved: its class name (`task.type`) + its row (`task.data`)."""
     runtime: RuntimeInfo | None = None
     """The runtime's full config plus its provisioned resource ID."""
+    version: int = TRACE_VERSION
+    """The trace record schema this trace serializes as."""
+    verifiers: VersionInfo | None = None
+    """The verifiers build that produced this trace, stamped at rollout start —
+    replayed/re-read traces keep the build that originally produced them."""
+    run: RunInfo | None = None
+    """The run this trace belongs to (eval or train), consumer-stamped."""
+    agent: AgentInfo | None = None
+    """The agent (model, sampling, harness) that produced the sampled turns."""
     nodes: list[MessageNode] = Field(default_factory=list)
     """The message graph; branches are derived views and storage stays linear in turns."""
     tools: list[Tool] | None = None
@@ -414,6 +478,13 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
             )
         self.rewards[name] = contribution
 
+    def stamp(self, run: RunInfo | None = None, **info: Any) -> None:
+        """Stamp identity only the consumer knows (the eval CLI / a trainer) onto the
+        trace; anything beyond `run` lands in `info`."""
+        if run is not None:
+            self.run = run
+        self.info.update(info)
+
     def stop(self, condition: str = "done") -> None:
         self.is_completed = True
         if self.stop_condition is None:

verifiers-0.2.1.dev57/verifiers/v1/utils/version.py

@@ -0,0 +1,43 @@
+"""Build provenance: the installed verifiers package's git commit, when resolvable."""
+
+import json
+import subprocess
+from functools import cache
+from importlib.metadata import PackageNotFoundError, distribution
+from pathlib import Path
+
+
+@cache
+def verifiers_commit() -> str | None:
+    """The git commit of the installed verifiers, when resolvable: a git-pinned
+    install records it in `direct_url.json` (`vcs_info.commit_id`); an editable or
+    source checkout answers `git rev-parse`. None otherwise (e.g. a PyPI wheel)."""
+    try:
+        direct_url = distribution("verifiers").read_text("direct_url.json")
+        if direct_url:
+            commit = json.loads(direct_url).get("vcs_info", {}).get("commit_id")
+            if commit:
+                return commit
+    except (PackageNotFoundError, ValueError):
+        pass
+    package_dir = Path(__file__).resolve().parents[2]
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(package_dir), "rev-parse", "--show-toplevel", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if result.returncode != 0:
+        return None
+    lines = result.stdout.splitlines()
+    if len(lines) != 2:
+        return None
+    toplevel, commit = lines
+    # git discovers `.git` upward, so an installed package in a venv inside an
+    # unrelated git project would answer with that project's repo.
+    if Path(toplevel).resolve() != package_dir.parent:
+        return None
+    return commit or None