verifiers 0.3.1.dev25__tar.gz → 0.3.1.dev32__tar.gz

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev25
+Version: 0.3.1.dev32
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/pyproject.toml

@@ -76,10 +76,23 @@ dev = [
 # from this repo. The built-in tasksets/harnesses ship inside verifiers (verifiers/v1/...).
 examples = [
     "compact",
-    "gsm8k", "glossary", "deepwiki", "wiki-search", "code-golf",
-    "proposer-solver", "reverse-text", "color-codeword",
-    "wordle", "openenv-wordle", "alphabet-sort", "scratchpad",
+    "gsm8k",
+    "glossary",
+    "deepwiki",
+    "wiki-search",
+    "code-golf",
+    "proposer-solver",
+    "reverse-text",
+    "color-codeword",
+    "wordle",
+    "openenv-wordle",
+    "alphabet-sort",
+    "scratchpad",
     "kuhn-poker",
+    "interception",
+    "grayscale-interception",
+    "bash-interception",
+    "web-search-interception",
 ]
 
 [project.optional-dependencies]
@@ -164,6 +177,10 @@ color-codeword = { path = "environments/color_codeword", editable = true }
 alphabet-sort = { path = "environments/alphabet_sort", editable = true }
 scratchpad = { path = "environments/scratchpad", editable = true }
 kuhn-poker = { path = "environments/kuhn_poker", editable = true }
+interception = { path = "environments/interception", editable = true }
+grayscale-interception = { path = "environments/grayscale_interception", editable = true }
+bash-interception = { path = "environments/bash_interception", editable = true }
+web-search-interception = { path = "environments/web_search_interception", editable = true }
 
 [tool.uv.exclude-newer-package]
 # Bounded cutoffs for the explicitly requested tool upgrades.

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/tests/v1/test_scoring.py

@@ -5,6 +5,9 @@ from verifiers.v1.graph import MessageNode
 from verifiers.v1.types import AssistantMessage, UserMessage
 
 PLUGGED_FNS_PY = """
+from verifiers.v1 import Trace
+
+
 async def reply_length(trace) -> float:
     return float(len(trace.last_reply or ""))
 
@@ -17,7 +20,7 @@ async def marker(trace) -> float:
     return 0.125
 
 
-async def two_turns(trace) -> bool:
+async def two_turns(trace: Trace) -> bool:
     return trace.num_turns >= 2
 """
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/__init__.py

@@ -80,6 +80,7 @@ from verifiers.v1.trace import (
     Branch,
     Error,
     EvalRunInfo,
+    InterceptRecord,
     ModelCall,
     PolicyEvent,
     Reward,
@@ -103,6 +104,7 @@ from verifiers.v1.types import (
     Message,
     MessageContent,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -121,7 +123,13 @@ from verifiers.v1.utils.artifacts import (
     collect,
     restore,
 )
-from verifiers.v1.utils.decorators import metric, reward, stop, tool
+from verifiers.v1.utils.decorators import (
+    intercept,
+    metric,
+    reward,
+    stop,
+    tool,
+)
 from verifiers.v1.utils.git import (
     PATCH_CAP_BYTES as PATCH_CAP_BYTES,
 )
@@ -178,6 +186,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Message",
     "MessageContent",
     "Messages",
+    "Request",
     "Response",
     "Sampling",
     "SamplingConfig",
@@ -220,10 +229,13 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "AgentSpan",
     "Error",
     # decorators
+    "intercept",
     "stop",
     "tool",
     "metric",
     "reward",
+    # interception
+    "InterceptRecord",
     # errors
     "RolloutError",
     "EnvError",

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/agent.py

@@ -465,15 +465,11 @@ class Agent:
         interaction = Interaction(run, gate=self._gate)
         async with self._gate or nullcontext():
             opened = await run.open()
-            if not opened:
+            if not opened and (failure := run.failure) is not None:
                 trace = await run.close()
                 if trace.agent.runtime is not None:
                     trace.agent.runtime.borrowed = runtime is not None
-        if not opened:
-            failure = run.failure
-            if failure is None:  # `open()` returning False always captures one.
-                raise RuntimeError("rollout setup failed without a captured error")
-            raise failure
+                raise failure
         try:
             yield interaction
         except Exception as e:

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/clients/train.py

@@ -342,7 +342,9 @@ class TrainClient(Client):
             prompt = turn.prompt
             tools = parse_tools(body.get("tools"))
         else:
-            prompt, tools = dialect.parse_request(body)
+            request = dialect.parse_request(body)
+            prompt = request.messages
+            tools = request.tools
         from renderers.client import generate
 
         wire_tools = [tool_to_wire(t) for t in tools] if tools else None

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/configs/agent.py

@@ -6,7 +6,7 @@ from pydantic_config import BaseConfig
 from verifiers.v1.clients import ClientConfig
 from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
-from verifiers.v1.runtimes import RuntimeConfig, SubprocessConfig
+from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 from verifiers.v1.types import SamplingConfig
 
 
@@ -26,7 +26,7 @@ class TimeoutConfig(BaseConfig):
 class AgentConfig(BaseConfig):
     harness: SerializeAsAny[HarnessConfig] | None = None
     """The agent's program (None = the taskset's default harness)."""
-    runtime: RuntimeConfig = SubprocessConfig()
+    runtime: RuntimeConfig = PrimeConfig()
     """Runtime for the harness program — the policy each run provisions its box
     from; tool servers choose their placement separately."""
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/configs/cli/debug.py

@@ -8,14 +8,14 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.cli.validate import CheckTimeoutConfig
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
+from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
 
 class DebugConfig(BaseConfig):
     uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
     """Auto-generated run id, used as the default output directory leaf."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
-    runtime: RuntimeConfig = DockerConfig()
+    runtime: RuntimeConfig = PrimeConfig()
     """Where each task's setup hook and debug action run."""
     command: str | None = None
     """Inline shell command executed as `sh -lc <command>` after setup."""

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/configs/cli/validate.py

@@ -7,7 +7,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
+from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
 
 class CheckTimeoutConfig(BaseConfig):
@@ -23,7 +23,7 @@ class ValidateConfig(BaseConfig):
     """Auto-generated run id — the default output directory leaf. Excluded from the
     saved config so re-running it starts a fresh run."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
-    runtime: RuntimeConfig = DockerConfig()
+    runtime: RuntimeConfig = PrimeConfig()
     """Where each task's validation hooks run."""
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     only_setup: bool = False

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/configs/client.py

@@ -18,7 +18,7 @@ from pydantic import Field, model_validator
 from pydantic_config import BaseConfig
 from renderers import RendererConfig
 
-from verifiers.v1.utils.prime_config import load_prime_config
+from verifiers.v1.utils.prime import load_prime_config
 
 DEFAULT_PRIME_INFERENCE_URL = "https://api.pinference.ai/api/v1"
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/configs/task.py

@@ -40,9 +40,9 @@ class TaskConfig(BaseConfig):
     """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
 
     stops: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
-    """Stop conditions `(trace) -> bool` plugged by name, merged with the task's
-    `@vf.stop` methods (a plugged function replaces a decorated one with the same
-    name)."""
+    """Typed `Request`, `Response`, or `Trace` stop predicates plugged by name and
+    merged with the task's `@vf.stop` methods (a plugged function replaces a
+    decorated one with the same name)."""
     metrics: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
     """Metrics `(task, trace, runtime) -> float` plugged by name, merged with the
     task's `@vf.metric` methods."""

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/dialects/anthropic.py

@@ -30,6 +30,7 @@ from verifiers.v1.types import (
     ImageUrlContentPart,
     ImageUrlSource,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -198,6 +199,37 @@ def mediate_content(value, path: str):
     return mediated, capabilities
 
 
+def content_to_wire(content) -> str | list[dict]:
+    """Typed text/image content in Anthropic's native request shape."""
+    if isinstance(content, str):
+        return content
+    blocks = []
+    for part in content:
+        if isinstance(part, TextContentPart):
+            blocks.append({"type": "text", "text": part.text})
+            continue
+        metadata, separator, data = part.image_url.url.partition(",")
+        if separator and metadata.startswith("data:") and metadata.endswith(";base64"):
+            blocks.append(
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": metadata[5:-7],
+                        "data": data,
+                    },
+                }
+            )
+        else:
+            blocks.append(
+                {
+                    "type": "image",
+                    "source": {"type": "url", "url": part.image_url.url},
+                }
+            )
+    return blocks
+
+
 def parse_messages(body: dict) -> Messages:
     """The request's top-level `system` + `messages` -> typed messages. Assistant turns fold
     their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's
@@ -380,7 +412,9 @@ class AnthropicStreamParser(StreamParser):
         for index, parts in self.partial_json.items():
             self.blocks[index]["input"] = json.loads("".join(parts) or "{}")
         self.message["content"] = [self.blocks[index] for index in sorted(self.blocks)]
-        return response_from_wire(self.validate_response(self.message))
+        response = response_from_wire(self.validate_response(self.message))
+        response.raw = self.message
+        return response
 
 
 class ModdedUsage(AnthropicUsage):
@@ -480,6 +514,12 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
         append_user_notice(mediated.setdefault("messages", []))
         return mediated, capabilities
 
+    def is_terminal_event(self, chunk: bytes) -> bool:
+        return any(
+            line.removeprefix(b"event:").strip() == b"message_stop"
+            for line in chunk.splitlines()
+        )
+
     def auth_headers(self, api_key: str) -> dict[str, str]:
         return {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
 
@@ -493,9 +533,7 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
             "error": {"type": "invalid_request_error", "message": message},
         }
 
-    def parse_request(
-        self, body: MessageCreateParams
-    ) -> tuple[Messages, list[Tool] | None]:
+    def parse_request(self, body: MessageCreateParams) -> Request:
         tools = [
             Tool(
                 name=t["name"],
@@ -505,11 +543,104 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
             for t in body.get("tools") or []
             if "input_schema" in t  # skip server tools (web_search etc.)
         ] or None
-        return parse_messages(body), tools
+        return Request(messages=parse_messages(body), tools=tools)
 
     def parse_response(self, response: AnthropicMessage) -> Response:
         return response_from_wire(response)
 
+    def rewrite_request(self, body: dict, before: Request, after: Request) -> None:
+        original = [
+            m for m in before.messages if isinstance(m, (UserMessage, ToolMessage))
+        ]
+        rewritten = [
+            m for m in after.messages if isinstance(m, (UserMessage, ToolMessage))
+        ]
+        targets: list[tuple[dict, dict | None]] = []
+        for native in body.get("messages", []):
+            if native.get("role") == "assistant":
+                continue
+            content = native.get("content")
+            if isinstance(content, str):
+                targets.append((native, None))
+                continue
+            blocks = content or []
+            targets.extend(
+                (native, block)
+                for block in blocks
+                if block.get("type") == "tool_result"
+            )
+            if any(block.get("type") != "tool_result" for block in blocks):
+                targets.append((native, None))
+
+        for (native, block), old, new in zip(targets, original, rewritten, strict=True):
+            if old == new:
+                continue
+            if block is not None:
+                block["content"] = content_to_wire(new.content)
+                continue
+            replacement = content_to_wire(new.content)
+            if isinstance(native.get("content"), str):
+                native["content"] = replacement
+                continue
+            replacement = (
+                [{"type": "text", "text": replacement}]
+                if isinstance(replacement, str)
+                else replacement
+            )
+            blocks = native.get("content") or []
+            updated = []
+            inserted = False
+            for current in blocks:
+                if current.get("type") == "tool_result":
+                    updated.append(current)
+                elif not inserted:
+                    updated.extend(replacement)
+                    inserted = True
+            native["content"] = updated
+
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        raw["content"] = [{"type": "text", "text": text}]
+        raw["stop_reason"] = "end_turn"
+        raw["stop_sequence"] = None
+
+    def stream_events(self, raw: dict) -> list[bytes]:
+        def event(kind: str, payload: dict) -> bytes:
+            return f"event: {kind}\ndata: {json.dumps(payload)}\n\n".encode()
+
+        text = raw["content"][0]["text"]
+        head = {**raw, "content": [], "stop_reason": None, "stop_sequence": None}
+        if isinstance(usage := head.get("usage"), dict):
+            head["usage"] = {**usage, "output_tokens": 0}
+        return [
+            event("message_start", {"type": "message_start", "message": head}),
+            event(
+                "content_block_start",
+                {
+                    "type": "content_block_start",
+                    "index": 0,
+                    "content_block": {"type": "text", "text": ""},
+                },
+            ),
+            event(
+                "content_block_delta",
+                {
+                    "type": "content_block_delta",
+                    "index": 0,
+                    "delta": {"type": "text_delta", "text": text},
+                },
+            ),
+            event("content_block_stop", {"type": "content_block_stop", "index": 0}),
+            event(
+                "message_delta",
+                {
+                    "type": "message_delta",
+                    "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+                    "usage": raw.get("usage") or {},
+                },
+            ),
+            event("message_stop", {"type": "message_stop"}),
+        ]
+
     def stream_parser(self) -> StreamParser:
         return AnthropicStreamParser(self.validate_response)
 

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/dialects/base.py

@@ -20,7 +20,7 @@ from typing import ClassVar, Generic, TypeVar
 from pydantic import BaseModel
 from pydantic_core import from_json
 
-from verifiers.v1.types import Messages, Response, Sampling, SamplingConfig, Tool
+from verifiers.v1.types import Request, Response, Sampling, SamplingConfig
 
 ReqT = TypeVar("ReqT")
 RespT = TypeVar("RespT", bound=BaseModel)
@@ -193,8 +193,8 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         content. Returned paths never contain request values."""
 
     @abstractmethod
-    def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]:
-        """The native request -> vf prompt + tools (for the trace)."""
+    def parse_request(self, body: ReqT) -> Request:
+        """The native request -> the typed model request."""
 
     def parse_sampling(self, body: ReqT) -> Sampling:
         """The native request's call settings -> the canonical `Sampling` (for the
@@ -212,6 +212,18 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         """Validate a native response, normalizing provider-compatible extensions if needed."""
         return self.response_type.model_validate(raw)
 
+    @abstractmethod
+    def rewrite_request(self, body: ReqT, before: Request, after: Request) -> None:
+        """Patch rewritten user/tool messages into the native conversation."""
+
+    @abstractmethod
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        """Replace the native assistant response with inert text."""
+
+    @abstractmethod
+    def stream_events(self, raw: dict) -> list[bytes]:
+        """Serialize a rewritten response as a minimal native SSE stream."""
+
     @abstractmethod
     def stream_parser(self) -> StreamParser:
         """Create the per-request incremental parser for a native SSE response."""

{verifiers-0.3.1.dev25 → verifiers-0.3.1.dev32}/verifiers/v1/dialects/chat.py

@@ -6,6 +6,7 @@ and responses (`parse_response`). Reasoning extraction mirrors the v0 chat clien
 read them in the same precedence (`reasoning` / `reasoning_content` / `reasoning_details`).
 """
 
+import json
 import time
 from collections.abc import Mapping
 from dataclasses import dataclass
@@ -28,6 +29,7 @@ from verifiers.v1.types import (
     FinishReason,
     Message,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -319,7 +321,9 @@ class ChatStreamParser(StreamParser):
             ],
             "usage": self.usage,
         }
-        return response_from_wire(ModdedChatCompletion.model_validate(completion))
+        response = response_from_wire(ModdedChatCompletion.model_validate(completion))
+        response.raw = completion
+        return response
 
 
 class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
@@ -473,9 +477,9 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
         append_user_notice(mediated.setdefault("messages", []))
         return mediated, capabilities
 
-    def parse_request(
-        self, body: CompletionCreateParams
-    ) -> tuple[Messages, list[Tool] | None]:
+    def parse_request(self, body: CompletionCreateParams) -> Request:
+        if body.get("n", 1) != 1:
+            raise ValueError("chat completions require n=1")
         messages: Messages = []
         tool_names: dict[str, str] = {}
         for raw in body.get("messages", []):
@@ -488,7 +492,7 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
             if isinstance(message, AssistantMessage):
                 for call in message.tool_calls or []:
                     tool_names[call.id] = call.name
-        return messages, parse_tools(body.get("tools"))
+        return Request(messages=messages, tools=parse_tools(body.get("tools")))
 
     def parse_sampling(self, body: CompletionCreateParams) -> Sampling:
         settings = {k: v for k, v in body.items() if k in self.sampling_fields}
@@ -501,6 +505,41 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
     def parse_response(self, response: ChatCompletion) -> Response:
         return response_from_wire(response)
 
+    def rewrite_request(self, body: dict, before: Request, after: Request) -> None:
+        for native, original, rewritten in zip(
+            body.get("messages", []), before.messages, after.messages, strict=True
+        ):
+            if rewritten != original:
+                native["content"] = message_to_wire(rewritten)["content"]
+                if isinstance(rewritten, ToolMessage):
+                    if rewritten.name is None:
+                        native.pop("name", None)
+                    else:
+                        native["name"] = rewritten.name
+
+    def rewrite_response(self, raw: dict, text: str) -> None:
+        for choice in raw.get("choices") or []:
+            if isinstance(choice.get("message"), dict):
+                choice["message"] = {"role": "assistant", "content": text}
+                choice["finish_reason"] = "stop"
+                choice.pop("logprobs", None)
+
+    def stream_events(self, raw: dict) -> list[bytes]:
+        choice = (raw.get("choices") or [{}])[0]
+        chunk = {
+            **{key: value for key, value in raw.items() if key != "choices"},
+            "object": "chat.completion.chunk",
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": choice.get("message")
+                    or {"role": "assistant", "content": ""},
+                    "finish_reason": choice.get("finish_reason"),
+                }
+            ],
+        }
+        return [f"data: {json.dumps(chunk)}\n\n".encode(), b"data: [DONE]\n\n"]
+
     def stream_parser(self) -> StreamParser:
         return ChatStreamParser()