verifiers 0.4.1.dev46__tar.gz → 0.4.1.dev47__tar.gz

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev46
+Version: 0.4.1.dev47
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/tests/integration/grid.py

@@ -192,31 +192,32 @@ def grid(
     return cells
 
 
-def transports(harness: str) -> tuple[str, ...]:
-    """The wire formats the harness config can select (`transport`) besides its
-    default. Empty when the program speaks one format or picks its own."""
-    field = harness_config_type(harness).model_fields.get("transport")
-    if field is None:
-        return ()
-    return tuple(t for t in get_args(field.annotation) if t != field.default)
+def dialects(harness: str) -> tuple[str, ...]:
+    """The dialects the harness config can select besides its default. Empty when the
+    program speaks one dialect or picks its own."""
+    field = harness_config_type(harness).model_fields["dialect"]
+    return tuple(
+        value
+        for arg in get_args(field.annotation)
+        for value in get_args(arg) or (arg,)
+        if value != field.default
+    )
 
 
-def transport_grid(
+def dialect_grid(
     runtimes: Sequence[str] = ("prime",), *, needs: tuple[str, ...] = ()
 ) -> list:
-    """`grid` cells for every non-default `transport` each harness config can select."""
+    """`grid` cells for every non-default `dialect` each harness config can select."""
     cells = []
     for harness in HARNESSES:
         name = harness_id(harness)
-        for transport in transports(name):
-            for cell in grid(
-                [{"id": name, "transport": transport}], runtimes, needs=needs
-            ):
+        for dialect in dialects(name):
+            for cell in grid([{"id": name, "dialect": dialect}], runtimes, needs=needs):
                 cells.append(
                     pytest.param(
                         *cell.values,
                         marks=cell.marks,
-                        id=cell.id.replace(f"{name}-in-", f"{name}-{transport}-in-"),
+                        id=cell.id.replace(f"{name}-in-", f"{name}-{dialect}-in-"),
                     )
                 )
     return cells

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/tests/integration/test_multi_turn.py

@@ -1,14 +1,14 @@
 """One task, many turns: the model completes a shell task (echo-agentic: write a phrase
 to a file, checked in the runtime) across the harness x runtime grid, and on prime over
-every other `transport` a harness config can select. A turn the harness re-sends
+every other `dialect` a harness config can select. A turn the harness re-sends
 differently, beyond what its `replay_view` declares, forks the trace."""
 
 import pytest
-from grid import grid, mark, transport_grid
+from grid import dialect_grid, grid, mark
 
 MULTI_TURN = [
     *grid(needs=("shell",)),
-    *transport_grid(needs=("shell",)),
+    *dialect_grid(needs=("shell",)),
     pytest.param(
         "bash",
         {"type": "modal", "allow": []},

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/tests/unit/test_graph.py

@@ -305,6 +305,14 @@ def _replayed_trace(replay_view, sampled, replayed, dialect=None):
         ),
         (strip_text, {"content": "Calling it.\n\n"}, {"content": "Calling it."}),
         (strip_text, {"content": None}, {"reasoning_content": " "}),
+        (
+            strip_text,
+            {"content": "\n\n"},
+            {
+                "reasoning_content": " ",
+                "provider_state": [{"type": "thinking", "thinking": " "}],
+            },
+        ),
     ],
 )
 def test_replay_view_matches_cosmetically_rewritten_turns(view, sampled, replayed):

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/tests/unit/utils/test_loaders.py

@@ -29,10 +29,10 @@ def test_every_builtin_harness_loads() -> None:
 
 def test_harness_config_narrows_to_its_own_type() -> None:
     config = vf.AgentConfig.model_validate(
-        {"harness": {"id": "kimi-code", "transport": "responses"}}
+        {"harness": {"id": "kimi-code", "dialect": "responses"}}
     )
     assert type(config.harness) is harness_config_type("kimi-code")
-    assert config.harness.transport == "responses"
+    assert config.harness.dialect == "responses"
 
 
 def test_unknown_harness_names_the_builtins() -> None:

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/configs/harness.py

@@ -9,7 +9,7 @@ from typing import Annotated, Literal
 from pydantic import ConfigDict, Field, FiniteFloat
 from pydantic_config import BaseConfig
 
-from verifiers.v1.types import ID
+from verifiers.v1.types import ID, DialectName
 
 PinnedVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9._+-]+$")]
 """A release/tag a harness pins its program install to."""
@@ -52,6 +52,9 @@ class HarnessConfig(BaseConfig):
     tool_timeout: FiniteFloat = Field(600.0, gt=0)
     """Seconds a single MCP tool call may take; raise it for tools that boot a VM."""
     disabled_tools: list[str] | None = None
+    dialect: DialectName | Literal["auto"] = "auto"
+    """Model API wire format the program speaks. `auto` lets the program pick, e.g. from
+    the model. Each harness narrows this to the formats it can select."""
     skills: list[SkillSource] = Field(default_factory=list)
     """Host skill folders or `{runtime: path}` roots of skills inside the runtime.
     Sources are installed in order; later files override matching earlier files.

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/bash/harness.py

@@ -1,4 +1,5 @@
 import os
+from typing import Literal
 
 from pydantic import PositiveInt
 from pydantic_config import BaseConfig
@@ -38,6 +39,7 @@ class CompactionConfig(BaseConfig):
 
 
 class BashHarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     edit: bool = True
     """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
     `bash`. On by default; set `--env.agent.harness.edit false` for a bash-only agent."""

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/browser_use/harness.py

@@ -32,6 +32,7 @@ Finding elements: prefer the accessibility tree over screenshots. cdp("Accessibi
 
 
 class BrowserUseHarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     browser: Literal["chromium", "cdp"] = "chromium"
     """`chromium` launches locally; `cdp` attaches to `cdp_url` without owning it."""
 

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/claude_code/harness.py

@@ -1,6 +1,7 @@
 """Run Claude Code through the Claude Agent SDK ACP adapter."""
 
 import json
+from typing import Literal
 
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
@@ -32,6 +33,7 @@ touch {ready}
 
 
 class ClaudeCodeHarnessConfig(HarnessConfig):
+    dialect: Literal["anthropic_messages"] = "anthropic_messages"
     version: PinnedVersion = "2.1.278"
     """Claude Code release to install, pinned for reproducibility."""
 

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/codex/harness.py

@@ -7,6 +7,7 @@ import re
 import tomllib
 from collections import Counter
 from pathlib import Path
+from typing import Literal
 
 import tomli_w
 
@@ -48,6 +49,9 @@ GATE_HANDLER = {
 
 
 class CodexHarnessConfig(HarnessConfig):
+    # Codex accepts no other wire API:
+    # https://github.com/openai/codex/blob/rust-v0.155.1/codex-rs/model-provider-info/src/lib.rs#L61
+    dialect: Literal["responses"] = "responses"
     version: PinnedVersion = "0.155.1"
     """Codex release to install, pinned for reproducibility."""
     multi_agent: bool = False

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/deepseek/harness.py

@@ -2,6 +2,7 @@
 
 import json
 from pathlib import Path
+from typing import Literal
 
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
@@ -25,6 +26,9 @@ touch "$VF_DEEPSEEK_DIR/.ready"
 
 
 class DeepSeekHarnessConfig(HarnessConfig):
+    # The `deepseek-official` route speaks Anthropic messages:
+    # https://github.com/deepseek-ai/deepseek-harness/blob/dsh-v0.2.1-alpha.3/packages/llm/llm-deepseek/src/messages-api.ts#L14-L17
+    dialect: Literal["anthropic_messages"] = "anthropic_messages"
     version: PinnedVersion = "0.2.1-alpha.3"
     """DeepSeek CLI release, pinned to its shipped sdk-minimal composition."""
 

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/hermes_agent/harness.py

@@ -2,6 +2,7 @@
 
 import json
 from pathlib import Path
+from typing import Literal
 
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
@@ -32,6 +33,10 @@ class HermesAgentHarnessConfig(HarnessConfig):
     """Hermes Agent Git release tag to install, pinned for reproducibility."""
     use_bundled_skill: bool = False
     """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
+    # Over Responses it replays an earlier turn without its reasoning.
+    dialect: Literal["chat_completions", "anthropic_messages", "auto"] = "auto"
+    """`auto` lets Hermes pick the transport from the model name on eval clients, and
+    chat completions on others."""
 
 
 class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]):
@@ -94,7 +99,9 @@ class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]):
             "api_key": secret,
             "discover_models": False,
         }
-        if ctx.client.type == "eval":
+        if self.config.dialect != "auto":
+            provider["transport"] = self.config.dialect
+        elif ctx.client.type == "eval":
             provider["transport"] = "${HERMES_INTERCEPT_TRANSPORT}"
         config = {
             "model": model,

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -40,8 +40,7 @@ fi
 class KimiCodeHarnessConfig(HarnessConfig):
     version: PinnedVersion = "2.0.2"
     """Kimi Code release to install, pinned for reproducibility."""
-    transport: DialectName = "chat_completions"
-    """Model API transport."""
+    dialect: DialectName = "chat_completions"
 
 
 class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
@@ -87,10 +86,10 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
             "chat_completions": "openai",
             "responses": "openai_responses",
             "anthropic_messages": "anthropic",
-        }[self.config.transport]
+        }[self.config.dialect]
         base_url = (
             endpoint.removesuffix("/v1")
-            if self.config.transport == "anthropic_messages"
+            if self.config.dialect == "anthropic_messages"
             else endpoint
         )
         config = {

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/mini_swe_agent/harness.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Literal
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
@@ -11,6 +12,7 @@ PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 
 class MiniSWEAgentHarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     version: PinnedVersion = "2.4.6"
     """mini-swe-agent release to install, pinned for reproducibility."""
 

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/null/harness.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.harness import Harness
@@ -11,7 +13,7 @@ from verifiers.v1.trace import Trace
 
 
 class NullHarnessConfig(HarnessConfig):
-    pass
+    dialect: Literal["chat_completions"] = "chat_completions"
 
 
 class NullHarness(Harness[NullHarnessConfig]):

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/openclaw/harness.py

@@ -2,6 +2,7 @@
 
 import json
 import logging
+from typing import Literal
 
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
@@ -118,6 +119,8 @@ wait "$acp_pid"
 class OpenClawHarnessConfig(HarnessConfig):
     version: PinnedVersion = "2026.9.5"
     """OpenClaw release to install, pinned for reproducibility."""
+    dialect: Literal["auto"] = "auto"
+    """OpenClaw takes the transport from its model catalog."""
     use_bundled_skill: bool = True
     """Enable OpenClaw's bundled skill catalog in addition to uploaded harness skills."""
 

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/opencode/harness.py

@@ -30,8 +30,7 @@ npm install --prefix "$VF_OPENCODE_DIR" --no-audit --no-fund --omit=dev \
 class OpenCodeHarnessConfig(HarnessConfig):
     version: PinnedVersion = "2.0.26"
     """OpenCode 2 release to install, pinned for reproducibility."""
-    transport: DialectName = "chat_completions"
-    """Model API transport."""
+    dialect: DialectName = "chat_completions"
     disabled_tools: list[str] | None = None
     """OpenCode permission actions to deny, such as shell, read, or edit."""
 
@@ -87,7 +86,7 @@ class OpenCodeHarness(ACPHarness[OpenCodeHarnessConfig]):
             "chat_completions": "openai-compatible",
             "responses": "openai-compatible/responses",
             "anthropic_messages": "anthropic-compatible",
-        }[self.config.transport]
+        }[self.config.dialect]
         settings = {
             "model": "intercept/model",
             "providers": {

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/pi/harness.py

@@ -50,8 +50,7 @@ fi
 class PiHarnessConfig(HarnessConfig):
     version: PinnedVersion = "0.86.1"
     """Pi release to install, pinned for reproducibility."""
-    transport: DialectName = "chat_completions"
-    """Model API transport."""
+    dialect: DialectName = "chat_completions"
     supports_developer_role: bool | None = None
     """Override Pi's chat-completions role detection for custom model endpoints."""
 
@@ -118,10 +117,10 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
             "chat_completions": "openai-completions",
             "responses": "openai-responses",
             "anthropic_messages": "anthropic-messages",
-        }[self.config.transport]
+        }[self.config.dialect]
         base_url = (
             endpoint.removesuffix("/v1")
-            if self.config.transport == "anthropic_messages"
+            if self.config.dialect == "anthropic_messages"
             else endpoint
         )
         model_config = {
@@ -130,12 +129,12 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
             "input": ["text", "image"],
             **(
                 {"compat": {"sessionAffinityFormat": "openai-nosession"}}
-                if self.config.transport == "responses"
+                if self.config.dialect == "responses"
                 else {}
             ),
         }
         if (
-            self.config.transport == "chat_completions"
+            self.config.dialect == "chat_completions"
             and self.config.supports_developer_role is not None
         ):
             model_config["compat"] = {

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/prime_agent/harness.py

@@ -82,6 +82,8 @@ class PrimeAgentHarnessConfig(HarnessConfig):
 
     autonomous: bool = False
     """Enable Prime Agent's autonomous continuation loop."""
+    # Over Responses it replays an earlier turn without its reasoning.
+    dialect: Literal["chat_completions", "anthropic_messages"] = "chat_completions"
 
 
 class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
@@ -209,8 +211,15 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
         models = {
             "providers": {
                 PROVIDER: {
-                    "baseUrl": endpoint,
-                    "api": "openai-completions",
+                    # Its Anthropic provider appends `/v1/messages` to the base URL.
+                    "baseUrl": endpoint.removesuffix("/v1")
+                    if self.config.dialect == "anthropic_messages"
+                    else endpoint,
+                    "api": {
+                        "chat_completions": "openai-completions",
+                        "responses": "openai-responses",
+                        "anthropic_messages": "anthropic-messages",
+                    }[self.config.dialect],
                     "apiKey": KEY_VAR,
                     "models": [
                         {

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/rlm/harness.py

@@ -67,6 +67,7 @@ class CompactionConfig(BaseConfig):
 
 
 class RLMHarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     version: str = Field(
         default="77c3489d1a7f918b75c31e195480ed3556af60a5", min_length=1
     )

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -1,5 +1,6 @@
 import logging
 from pathlib import Path
+from typing import Literal
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
@@ -13,6 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 class Terminus2HarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     version: PinnedVersion = "0.23.0"
     """Harbor release to install, pinned for reproducibility."""
 

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/harnesses/utils/replay.py

@@ -19,7 +19,7 @@ def drop_blank_text(message: Message) -> Message:
 
 def strip_text(message: Message) -> Message:
     """Clients that store assistant text stripped, and pad a missing reasoning field
-    with a single space on replay."""
+    (or Anthropic thinking block) with a single space on replay."""
     if not isinstance(message, AssistantMessage):
         return message
     update: dict = {}
@@ -27,4 +27,14 @@ def strip_text(message: Message) -> Message:
         update["content"] = message.content.strip() or None
     if message.reasoning_content is not None and not message.reasoning_content.strip():
         update["reasoning_content"] = None
+    if message.provider_state and any(
+        _blank_thinking(item) for item in message.provider_state
+    ):
+        update["provider_state"] = [
+            item for item in message.provider_state if not _blank_thinking(item)
+        ] or None
     return message.model_copy(update=update) if update else message
+
+
+def _blank_thinking(item: dict) -> bool:
+    return item.get("type") == "thinking" and not str(item.get("thinking", "")).strip()

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/types.py

@@ -126,8 +126,8 @@ class Request(BaseModel):
 FinishReason = Literal["stop", "length", "tool_calls"] | None
 
 DialectName = Literal["chat_completions", "responses", "anthropic_messages"]
-"""A model API wire format: a `Dialect.name`, and the values of harness `transport`
-settings."""
+"""A model API wire format: a `Dialect.name`, and the values of the harness `dialect`
+setting."""
 
 
 class Usage(BaseModel):

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev47}/verifiers/v1/utils/loaders.py

@@ -52,7 +52,8 @@ def narrow_plugin_field(
 ) -> None:
     raw = data.get(field)
     if isinstance(raw, BaseConfig):
-        raw = raw.model_dump()
+        # Unset fields take the concrete type's defaults, which may narrow the base's.
+        raw = raw.model_dump(exclude_unset=True)
     raw = dict(raw or {})
     ident = raw.get("id") or default_id
     if not ident: