verifiers 0.3.1.dev25__tar.gz → 0.3.1.dev31__tar.gz

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev25
+Version: 0.3.1.dev31
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/pyproject.toml

@@ -76,10 +76,23 @@ dev = [
 # from this repo. The built-in tasksets/harnesses ship inside verifiers (verifiers/v1/...).
 examples = [
     "compact",
-    "gsm8k", "glossary", "deepwiki", "wiki-search", "code-golf",
-    "proposer-solver", "reverse-text", "color-codeword",
-    "wordle", "openenv-wordle", "alphabet-sort", "scratchpad",
+    "gsm8k",
+    "glossary",
+    "deepwiki",
+    "wiki-search",
+    "code-golf",
+    "proposer-solver",
+    "reverse-text",
+    "color-codeword",
+    "wordle",
+    "openenv-wordle",
+    "alphabet-sort",
+    "scratchpad",
     "kuhn-poker",
+    "interception",
+    "grayscale-interception",
+    "bash-interception",
+    "web-search-interception",
 ]
 
 [project.optional-dependencies]
@@ -164,6 +177,10 @@ color-codeword = { path = "environments/color_codeword", editable = true }
 alphabet-sort = { path = "environments/alphabet_sort", editable = true }
 scratchpad = { path = "environments/scratchpad", editable = true }
 kuhn-poker = { path = "environments/kuhn_poker", editable = true }
+interception = { path = "environments/interception", editable = true }
+grayscale-interception = { path = "environments/grayscale_interception", editable = true }
+bash-interception = { path = "environments/bash_interception", editable = true }
+web-search-interception = { path = "environments/web_search_interception", editable = true }
 
 [tool.uv.exclude-newer-package]
 # Bounded cutoffs for the explicitly requested tool upgrades.

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/tests/v1/test_scoring.py

@@ -5,6 +5,9 @@ from verifiers.v1.graph import MessageNode
 from verifiers.v1.types import AssistantMessage, UserMessage
 
 PLUGGED_FNS_PY = """
+from verifiers.v1 import Trace
+
+
 async def reply_length(trace) -> float:
     return float(len(trace.last_reply or ""))
 
@@ -17,7 +20,7 @@ async def marker(trace) -> float:
     return 0.125
 
 
-async def two_turns(trace) -> bool:
+async def two_turns(trace: Trace) -> bool:
     return trace.num_turns >= 2
 """
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/__init__.py

@@ -80,6 +80,7 @@ from verifiers.v1.trace import (
     Branch,
     Error,
     EvalRunInfo,
+    InterceptRecord,
     ModelCall,
     PolicyEvent,
     Reward,
@@ -103,6 +104,7 @@ from verifiers.v1.types import (
     Message,
     MessageContent,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -121,7 +123,13 @@ from verifiers.v1.utils.artifacts import (
     collect,
     restore,
 )
-from verifiers.v1.utils.decorators import metric, reward, stop, tool
+from verifiers.v1.utils.decorators import (
+    intercept,
+    metric,
+    reward,
+    stop,
+    tool,
+)
 from verifiers.v1.utils.git import (
     PATCH_CAP_BYTES as PATCH_CAP_BYTES,
 )
@@ -178,6 +186,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Message",
     "MessageContent",
     "Messages",
+    "Request",
     "Response",
     "Sampling",
     "SamplingConfig",
@@ -220,10 +229,13 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "AgentSpan",
     "Error",
     # decorators
+    "intercept",
     "stop",
     "tool",
     "metric",
     "reward",
+    # interception
+    "InterceptRecord",
     # errors
     "RolloutError",
     "EnvError",

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/agent.py

@@ -465,15 +465,11 @@ class Agent:
         interaction = Interaction(run, gate=self._gate)
         async with self._gate or nullcontext():
             opened = await run.open()
-            if not opened:
+            if not opened and (failure := run.failure) is not None:
                 trace = await run.close()
                 if trace.agent.runtime is not None:
                     trace.agent.runtime.borrowed = runtime is not None
-        if not opened:
-            failure = run.failure
-            if failure is None:  # `open()` returning False always captures one.
-                raise RuntimeError("rollout setup failed without a captured error")
-            raise failure
+                raise failure
         try:
             yield interaction
         except Exception as e:

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/clients/train.py

@@ -342,7 +342,9 @@ class TrainClient(Client):
             prompt = turn.prompt
             tools = parse_tools(body.get("tools"))
         else:
-            prompt, tools = dialect.parse_request(body)
+            request = dialect.parse_request(body)
+            prompt = request.messages
+            tools = request.tools
         from renderers.client import generate
 
         wire_tools = [tool_to_wire(t) for t in tools] if tools else None

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/configs/task.py

@@ -40,9 +40,9 @@ class TaskConfig(BaseConfig):
     """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
 
     stops: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
-    """Stop conditions `(trace) -> bool` plugged by name, merged with the task's
-    `@vf.stop` methods (a plugged function replaces a decorated one with the same
-    name)."""
+    """Typed `Request`, `Response`, or `Trace` stop predicates plugged by name and
+    merged with the task's `@vf.stop` methods (a plugged function replaces a
+    decorated one with the same name)."""
     metrics: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
     """Metrics `(task, trace, runtime) -> float` plugged by name, merged with the
     task's `@vf.metric` methods."""

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/dialects/anthropic.py

@@ -30,6 +30,7 @@ from verifiers.v1.types import (
     ImageUrlContentPart,
     ImageUrlSource,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -198,6 +199,37 @@ def mediate_content(value, path: str):
     return mediated, capabilities
 
 
+def content_to_wire(content) -> str | list[dict]:
+    """Typed text/image content in Anthropic's native request shape."""
+    if isinstance(content, str):
+        return content
+    blocks = []
+    for part in content:
+        if isinstance(part, TextContentPart):
+            blocks.append({"type": "text", "text": part.text})
+            continue
+        metadata, separator, data = part.image_url.url.partition(",")
+        if separator and metadata.startswith("data:") and metadata.endswith(";base64"):
+            blocks.append(
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": metadata[5:-7],
+                        "data": data,
+                    },
+                }
+            )
+        else:
+            blocks.append(
+                {
+                    "type": "image",
+                    "source": {"type": "url", "url": part.image_url.url},
+                }
+            )
+    return blocks
+
+
 def parse_messages(body: dict) -> Messages:
     """The request's top-level `system` + `messages` -> typed messages. Assistant turns fold
     their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's
@@ -380,7 +412,9 @@ class AnthropicStreamParser(StreamParser):
         for index, parts in self.partial_json.items():
             self.blocks[index]["input"] = json.loads("".join(parts) or "{}")
         self.message["content"] = [self.blocks[index] for index in sorted(self.blocks)]
-        return response_from_wire(self.validate_response(self.message))
+        response = response_from_wire(self.validate_response(self.message))
+        response.raw = self.message
+        return response
 
 
 class ModdedUsage(AnthropicUsage):
@@ -480,6 +514,12 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
         append_user_notice(mediated.setdefault("messages", []))
         return mediated, capabilities
 
+    def is_terminal_event(self, chunk: bytes) -> bool:
+        return any(
+            line.removeprefix(b"event:").strip() == b"message_stop"
+            for line in chunk.splitlines()
+        )
+
     def auth_headers(self, api_key: str) -> dict[str, str]:
         return {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
 
@@ -493,9 +533,7 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
             "error": {"type": "invalid_request_error", "message": message},
         }
 
-    def parse_request(
-        self, body: MessageCreateParams
-    ) -> tuple[Messages, list[Tool] | None]:
+    def parse_request(self, body: MessageCreateParams) -> Request:
         tools = [
             Tool(
                 name=t["name"],
@@ -505,11 +543,104 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
             for t in body.get("tools") or []
             if "input_schema" in t  # skip server tools (web_search etc.)
         ] or None
-        return parse_messages(body), tools
+        return Request(messages=parse_messages(body), tools=tools)
 
     def parse_response(self, response: AnthropicMessage) -> Response:
         return response_from_wire(response)
 
+    def rewrite_request(self, body: dict, before: Request, after: Request) -> None:
+        original = [
+            m for m in before.messages if isinstance(m, (UserMessage, ToolMessage))
+        ]
+        rewritten = [
+            m for m in after.messages if isinstance(m, (UserMessage, ToolMessage))
+        ]
+        targets: list[tuple[dict, dict | None]] = []
+        for native in body.get("messages", []):
+            if native.get("role") == "assistant":
+                continue
+            content = native.get("content")
+            if isinstance(content, str):
+                targets.append((native, None))
+                continue
+            blocks = content or []
+            targets.extend(
+                (native, block)
+                for block in blocks
+                if block.get("type") == "tool_result"
+            )
+            if any(block.get("type") != "tool_result" for block in blocks):
+                targets.append((native, None))
+
+        for (native, block), old, new in zip(targets, original, rewritten, strict=True):
+            if old == new:
+                continue
+            if block is not None:
+                block["content"] = content_to_wire(new.content)
+                continue
+            replacement = content_to_wire(new.content)
+            if isinstance(native.get("content"), str):
+                native["content"] = replacement
+                continue
+            replacement = (
+                [{"type": "text", "text": replacement}]
+                if isinstance(replacement, str)
+                else replacement
+            )
+            blocks = native.get("content") or []
+            updated = []
+            inserted = False
+            for current in blocks:
+                if current.get("type") == "tool_result":
+                    updated.append(current)
+                elif not inserted:
+                    updated.extend(replacement)
+                    inserted = True
+            native["content"] = updated
+
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        raw["content"] = [{"type": "text", "text": text}]
+        raw["stop_reason"] = "end_turn"
+        raw["stop_sequence"] = None
+
+    def stream_events(self, raw: dict) -> list[bytes]:
+        def event(kind: str, payload: dict) -> bytes:
+            return f"event: {kind}\ndata: {json.dumps(payload)}\n\n".encode()
+
+        text = raw["content"][0]["text"]
+        head = {**raw, "content": [], "stop_reason": None, "stop_sequence": None}
+        if isinstance(usage := head.get("usage"), dict):
+            head["usage"] = {**usage, "output_tokens": 0}
+        return [
+            event("message_start", {"type": "message_start", "message": head}),
+            event(
+                "content_block_start",
+                {
+                    "type": "content_block_start",
+                    "index": 0,
+                    "content_block": {"type": "text", "text": ""},
+                },
+            ),
+            event(
+                "content_block_delta",
+                {
+                    "type": "content_block_delta",
+                    "index": 0,
+                    "delta": {"type": "text_delta", "text": text},
+                },
+            ),
+            event("content_block_stop", {"type": "content_block_stop", "index": 0}),
+            event(
+                "message_delta",
+                {
+                    "type": "message_delta",
+                    "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+                    "usage": raw.get("usage") or {},
+                },
+            ),
+            event("message_stop", {"type": "message_stop"}),
+        ]
+
     def stream_parser(self) -> StreamParser:
         return AnthropicStreamParser(self.validate_response)
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/dialects/base.py

@@ -20,7 +20,7 @@ from typing import ClassVar, Generic, TypeVar
 from pydantic import BaseModel
 from pydantic_core import from_json
 
-from verifiers.v1.types import Messages, Response, Sampling, SamplingConfig, Tool
+from verifiers.v1.types import Request, Response, Sampling, SamplingConfig
 
 ReqT = TypeVar("ReqT")
 RespT = TypeVar("RespT", bound=BaseModel)
@@ -193,8 +193,8 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         content. Returned paths never contain request values."""
 
     @abstractmethod
-    def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]:
-        """The native request -> vf prompt + tools (for the trace)."""
+    def parse_request(self, body: ReqT) -> Request:
+        """The native request -> the typed model request."""
 
     def parse_sampling(self, body: ReqT) -> Sampling:
         """The native request's call settings -> the canonical `Sampling` (for the
@@ -212,6 +212,18 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         """Validate a native response, normalizing provider-compatible extensions if needed."""
         return self.response_type.model_validate(raw)
 
+    @abstractmethod
+    def rewrite_request(self, body: ReqT, before: Request, after: Request) -> None:
+        """Patch rewritten user/tool messages into the native conversation."""
+
+    @abstractmethod
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        """Replace the native assistant response with inert text."""
+
+    @abstractmethod
+    def stream_events(self, raw: dict) -> list[bytes]:
+        """Serialize a rewritten response as a minimal native SSE stream."""
+
     @abstractmethod
     def stream_parser(self) -> StreamParser:
         """Create the per-request incremental parser for a native SSE response."""

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/dialects/chat.py

@@ -6,6 +6,7 @@ and responses (`parse_response`). Reasoning extraction mirrors the v0 chat clien
 read them in the same precedence (`reasoning` / `reasoning_content` / `reasoning_details`).
 """
 
+import json
 import time
 from collections.abc import Mapping
 from dataclasses import dataclass
@@ -28,6 +29,7 @@ from verifiers.v1.types import (
     FinishReason,
     Message,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -319,7 +321,9 @@ class ChatStreamParser(StreamParser):
             ],
             "usage": self.usage,
         }
-        return response_from_wire(ModdedChatCompletion.model_validate(completion))
+        response = response_from_wire(ModdedChatCompletion.model_validate(completion))
+        response.raw = completion
+        return response
 
 
 class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
@@ -473,9 +477,9 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
         append_user_notice(mediated.setdefault("messages", []))
         return mediated, capabilities
 
-    def parse_request(
-        self, body: CompletionCreateParams
-    ) -> tuple[Messages, list[Tool] | None]:
+    def parse_request(self, body: CompletionCreateParams) -> Request:
+        if body.get("n", 1) != 1:
+            raise ValueError("chat completions require n=1")
         messages: Messages = []
         tool_names: dict[str, str] = {}
         for raw in body.get("messages", []):
@@ -488,7 +492,7 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
             if isinstance(message, AssistantMessage):
                 for call in message.tool_calls or []:
                     tool_names[call.id] = call.name
-        return messages, parse_tools(body.get("tools"))
+        return Request(messages=messages, tools=parse_tools(body.get("tools")))
 
     def parse_sampling(self, body: CompletionCreateParams) -> Sampling:
         settings = {k: v for k, v in body.items() if k in self.sampling_fields}
@@ -501,6 +505,41 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
     def parse_response(self, response: ChatCompletion) -> Response:
         return response_from_wire(response)
 
+    def rewrite_request(self, body: dict, before: Request, after: Request) -> None:
+        for native, original, rewritten in zip(
+            body.get("messages", []), before.messages, after.messages, strict=True
+        ):
+            if rewritten != original:
+                native["content"] = message_to_wire(rewritten)["content"]
+                if isinstance(rewritten, ToolMessage):
+                    if rewritten.name is None:
+                        native.pop("name", None)
+                    else:
+                        native["name"] = rewritten.name
+
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        for choice in raw.get("choices") or []:
+            if isinstance(choice.get("message"), dict):
+                choice["message"] = {"role": "assistant", "content": text}
+                choice["finish_reason"] = "stop"
+                choice.pop("logprobs", None)
+
+    def stream_events(self, raw: dict) -> list[bytes]:
+        choice = (raw.get("choices") or [{}])[0]
+        chunk = {
+            **{key: value for key, value in raw.items() if key != "choices"},
+            "object": "chat.completion.chunk",
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": choice.get("message")
+                    or {"role": "assistant", "content": ""},
+                    "finish_reason": choice.get("finish_reason"),
+                }
+            ],
+        }
+        return [f"data: {json.dumps(chunk)}\n\n".encode(), b"data: [DONE]\n\n"]
+
     def stream_parser(self) -> StreamParser:
         return ChatStreamParser()
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/dialects/responses.py

@@ -32,6 +32,7 @@ from verifiers.v1.types import (
     ImageUrlContentPart,
     ImageUrlSource,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -386,9 +387,11 @@ class ResponsesStreamParser(StreamParser):
         events = self.terminal_events or self.events
         for event in iter_sse_reverse(b"".join(events)):
             if event.get("type") in FINAL_EVENTS:
-                return response_from_wire(
+                response = response_from_wire(
                     OpenAIResponse.model_validate(event["response"])
                 )
+                response.raw = event["response"]
+                return response
         raise ValueError("Responses stream ended without a terminal event")
 
 
@@ -541,9 +544,7 @@ class ResponsesDialect(Dialect[ResponseCreateParams, OpenAIResponse]):
             settings["max_tokens"] = settings.pop("max_output_tokens")
         return Sampling.model_validate(settings)
 
-    def parse_request(
-        self, body: ResponseCreateParams
-    ) -> tuple[Messages, list[Tool] | None]:
+    def parse_request(self, body: ResponseCreateParams) -> Request:
         prompt: Messages = []
         if instructions := body.get("instructions"):
             prompt.append(SystemMessage(content=instructions))
@@ -596,11 +597,127 @@ class ResponsesDialect(Dialect[ResponseCreateParams, OpenAIResponse]):
             for t in body.get("tools") or []
             if t.get("type") == "function"
         ] or None
-        return prompt, tools
+        return Request(messages=prompt, tools=tools)
 
     def parse_response(self, response: OpenAIResponse) -> Response:
         return response_from_wire(response)
 
+    def rewrite_request(self, body: dict, before: Request, after: Request) -> None:
+        original = [
+            m for m in before.messages if isinstance(m, (UserMessage, ToolMessage))
+        ]
+        rewritten = [
+            m for m in after.messages if isinstance(m, (UserMessage, ToolMessage))
+        ]
+        items = body.get("input")
+        if isinstance(items, str):
+            if original != rewritten:
+                message = rewritten[0]
+                content = (
+                    message.content
+                    if isinstance(message.content, str)
+                    else [
+                        {"type": "input_text", "text": part.text}
+                        if isinstance(part, TextContentPart)
+                        else {
+                            "type": "input_image",
+                            "image_url": part.image_url.url,
+                        }
+                        for part in message.content
+                    ]
+                )
+                body["input"] = (
+                    content
+                    if isinstance(content, str)
+                    else [{"role": "user", "content": content}]
+                )
+            return
+
+        targets = []
+        for item in items or []:
+            role = item.get("role")
+            kind = item.get("type") or ""
+            assistant = role == "assistant" or (
+                role is None and not kind.endswith(("_output", "_response"))
+            )
+            if assistant or role in ("system", "developer"):
+                continue
+            targets.append(item)
+        for item, old, new in zip(targets, original, rewritten, strict=True):
+            if old == new:
+                continue
+            content = (
+                new.content
+                if isinstance(new.content, str)
+                else [
+                    {"type": "input_text", "text": part.text}
+                    if isinstance(part, TextContentPart)
+                    else {
+                        "type": "input_image",
+                        "image_url": part.image_url.url,
+                    }
+                    for part in new.content
+                ]
+            )
+            item["output" if isinstance(new, ToolMessage) else "content"] = content
+
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        original = next(
+            (
+                item
+                for item in raw.get("output") or []
+                if isinstance(item, dict) and item.get("type") == "message"
+            ),
+            {},
+        )
+        raw["output"] = [
+            {
+                "type": "message",
+                "id": original.get("id") or "msg_intercepted",
+                "role": "assistant",
+                "status": "completed",
+                "content": [{"type": "output_text", "text": text, "annotations": []}],
+            }
+        ]
+        raw.update(status="completed", error=None, incomplete_details=None)
+        raw.pop("required_action", None)
+        if "output_text" in raw:
+            raw["output_text"] = text
+
+    def stream_events(self, raw: dict) -> list[bytes]:
+        item = raw["output"][0]
+        part = item["content"][0]
+        common = {"output_index": 0, "item_id": item["id"], "content_index": 0}
+        head = {
+            **raw,
+            "status": "in_progress",
+            "output": [],
+            "completed_at": None,
+        }
+        events = [
+            ("response.created", {"response": head}),
+            (
+                "response.output_item.added",
+                {"output_index": 0, "item": {**item, "content": []}},
+            ),
+            (
+                "response.content_part.added",
+                {**common, "part": {**part, "text": ""}},
+            ),
+            ("response.output_text.delta", {**common, "delta": part["text"]}),
+            ("response.output_text.done", {**common, "text": part["text"]}),
+            ("response.content_part.done", {**common, "part": part}),
+            ("response.output_item.done", {"output_index": 0, "item": item}),
+            ("response.completed", {"response": raw}),
+        ]
+        return [
+            *(
+                f"data: {json.dumps({'type': kind, 'sequence_number': i, **data})}\n\n".encode()
+                for i, (kind, data) in enumerate(events)
+            ),
+            b"data: [DONE]\n\n",
+        ]
+
     def stream_parser(self) -> StreamParser:
         return ResponsesStreamParser()
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev31}/verifiers/v1/graph.py

@@ -375,6 +375,18 @@ class PendingTurn:
             self.trace.tools = tools
         return assistant_id
 
+    def commit_prompt(self, tools: list[Tool] | None = None) -> None:
+        """Record an input that terminated before model inference."""
+        parent = self.prefix_node_ids[-1] if self.prefix_node_ids else None
+        index = _head_index(self.trace)
+        for message in self.tail:
+            previous = parent
+            self.trace.nodes.append(MessageNode(parent=parent, message=message))
+            parent = len(self.trace.nodes) - 1
+            index[(previous, message_hash(message))] = parent
+        if tools:
+            self.trace.tools = tools
+
 
 def prepare_turn(trace: Trace, prompt: list[Message]) -> PendingTurn:
     """Resolve `prompt` against the trace graph without mutating it."""