verifiers 0.4.1.dev30__tar.gz → 0.4.1.dev32__tar.gz

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev30
+Version: 0.4.1.dev32
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/pyproject.toml

@@ -237,6 +237,7 @@ markers = [
     "claude_code: integration cells on the claude-code harness",
     "openclaw: integration cells on the OpenClaw harness",
     "hermes_agent: integration cells on the hermes-agent harness",
+    "deepseek: integration cells on the DeepSeek minimal harness",
     "rlm: integration cells on the rlm harness",
     "kimi_code: integration cells on the kimi-code harness",
     "pi: integration cells on the pi harness",

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/tests/integration/grid.py

@@ -49,6 +49,7 @@ HARNESSES: list[str | dict] = [
     "codex",
     "claude-code",
     "hermes-agent",
+    "deepseek",
     "kimi-code",
     "pi",
     "openclaw",

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/verifiers/v1/dialects/anthropic.py

@@ -24,6 +24,7 @@ from verifiers.v1.dialects.base import (
     parse_sse_event,
     provider_domains,
 )
+from verifiers.v1.errors import ProviderError
 from verifiers.v1.types import (
     AssistantMessage,
     ContentPart,
@@ -51,6 +52,18 @@ STOP_REASONS: dict[str, FinishReason] = {
     "tool_use": "tool_calls",
     "stop_sequence": "stop",
 }
+ERROR_TYPES = {
+    400: "invalid_request_error",
+    401: "authentication_error",
+    403: "permission_error",
+    404: "not_found_error",
+    413: "request_too_large",
+    429: "rate_limit_error",
+    502: "api_error",
+    529: "overloaded_error",
+}
+ERROR_STATUSES = {kind: status for status, kind in ERROR_TYPES.items()}
+
 # Claude may reorder mixed thinking block types between a response and its replay.
 # Native tool events share the final rank, preserving their relative order.
 THINKING_ORDER = {"redacted_thinking": 0, "thinking": 1}
@@ -356,6 +369,11 @@ class AnthropicStreamParser(StreamParser):
         if event is None:
             return
         kind = event.get("type")
+        if kind == "error":
+            # HTTP 200 streams can fail in-band; preserve the provider's cause and IDs.
+            detail = event.get("error") or {}
+            status = ERROR_STATUSES.get(detail.get("type"), 502)
+            raise ProviderError(json.dumps(event), status_code=status)
         if kind == "message_start":
             self.message = event.get("message") or {}
         elif kind == "content_block_start":
@@ -516,7 +534,14 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
         # Anthropic's own keepalive; its SDKs skip it anywhere in the stream.
         return b'event: ping\ndata: {"type": "ping"}\n\n'
 
-    def stream_error(self, error: dict) -> bytes:
+    def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
+        # Once HTTP 200 is committed, clients classify retries from the SSE error type.
+        if status_code is not None:
+            kind = ERROR_TYPES.get(
+                status_code,
+                "api_error" if status_code >= 500 else "invalid_request_error",
+            )
+            error = {**error, "error": {**error.get("error", {}), "type": kind}}
         # The Anthropic SDKs raise only on a named `error` event.
         return b"event: error\ndata: " + json.dumps(error).encode() + b"\n\n"
 

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/verifiers/v1/dialects/base.py

@@ -318,7 +318,7 @@ class Dialect(ABC, Generic[RespT]):
         # Don't terminate an empty event; some SSE clients try to JSON-decode it.
         return b": keepalive\n"
 
-    def stream_error(self, error: dict) -> bytes:
+    def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
         """An `error_body` as an SSE event, for a failure after the stream is committed.
         OpenAI SDKs raise on any event carrying `error`."""
         return b"data: " + json.dumps(error).encode() + b"\n\n"

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/verifiers/v1/dialects/responses.py

@@ -687,7 +687,7 @@ class ResponsesDialect(Dialect[OpenAIResponse]):
         payload = {"type": kind, "sequence_number": 0, "response": _KEEPALIVE_RESPONSE}
         return f"event: {kind}\ndata: {json.dumps(payload)}\n\n".encode()
 
-    def stream_error(self, error: dict) -> bytes:
+    def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
         # `response.failed` is what Responses clients (codex) act on; the `error` key is what
         # the OpenAI SDK raises on.
         message = error["error"]["message"]

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/verifiers/v1/harnesses/__init__.py

@@ -8,6 +8,7 @@ from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
+from verifiers.v1.harnesses.deepseek import DeepSeekHarness, DeepSeekHarnessConfig
 from verifiers.v1.harnesses.hermes_agent import (
     HermesAgentHarness,
     HermesAgentHarnessConfig,
@@ -37,6 +38,8 @@ __all__ = [
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
+    "DeepSeekHarness",
+    "DeepSeekHarnessConfig",
     "HermesAgentHarness",
     "HermesAgentHarnessConfig",
     "KimiCodeHarness",

verifiers-0.4.1.dev32/verifiers/v1/harnesses/deepseek/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.deepseek.harness import (
+    DeepSeekHarness,
+    DeepSeekHarnessConfig,
+)
+
+__all__ = ["DeepSeekHarness", "DeepSeekHarnessConfig"]

verifiers-0.4.1.dev32/verifiers/v1/harnesses/deepseek/gate.mjs

@@ -0,0 +1,11 @@
+// Preserve native policy denials; route every otherwise allowed tool through ACP.
+// The shared ACP runner resolves the call ID against the rollout's tool policy.
+export const name = "vf-tool-gate";
+export const inject = ["tools", "approval"];
+
+export function apply(ctx) {
+  ctx.on("tools/pre-execute", async (_exec, next) => {
+    const decision = await next();
+    return decision.kind === "allow" ? { kind: "ask" } : decision;
+  });
+}

verifiers-0.4.1.dev32/verifiers/v1/harnesses/deepseek/harness.py

@@ -0,0 +1,148 @@
+"""DeepSeek's minimal coding agent, driven through its native ACP bridge."""
+
+import json
+from pathlib import Path
+
+from verifiers.v1.acp import ACPConfig, ACPHarness
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
+from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
+from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
+from verifiers.v1.runtimes import Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+DEEPSEEK_DIR = "/var/tmp/vf-deepseek-{version}"
+USAGE_COMPAT = (Path(__file__).parent / "usage.mjs").read_bytes()
+GATE = (Path(__file__).parent / "gate.mjs").read_bytes()
+INSTALL = r"""
+set -e
+export PATH="/var/tmp/vf-node/bin:$PATH"
+npm install --prefix "$VF_DEEPSEEK_DIR" --no-audit --no-fund \
+    "@deepseek-ai/dsh@$VF_DEEPSEEK_VERSION" >/dev/null
+touch "$VF_DEEPSEEK_DIR/.ready"
+"""
+
+
+class DeepSeekHarnessConfig(HarnessConfig):
+    version: PinnedVersion = "0.2.1-alpha.3"
+    """DeepSeek CLI release, pinned to its shipped sdk-minimal composition."""
+
+
+class DeepSeekHarness(ACPHarness[DeepSeekHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_TOOL_INTERCEPTION = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await ensure_node(runtime)
+        directory = DEEPSEEK_DIR.format(version=self.config.version)
+        await ensure_installed(
+            runtime,
+            directory=directory,
+            ready=f"test -f {directory}/.ready",
+            install=INSTALL,
+            env={
+                "VF_DEEPSEEK_DIR": directory,
+                "VF_DEEPSEEK_VERSION": self.config.version,
+            },
+            label="DeepSeek Harness",
+        )
+        await super().setup(runtime)
+
+    async def prepare_acp(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ACPConfig:
+        if self.config.disabled_tools:
+            raise ValueError("DeepSeek minimal ACP does not support disabling tools")
+
+        home = f"/tmp/vf-deepseek/{trace.id}"
+        system_prompt, prompt = self.resolve_prompt(data)
+        # Keep the upstream minimal tree; replace only its SDK stdio front door.
+        # https://github.com/deepseek-ai/deepseek-harness/tree/master/packages/bundle/sdk-minimal
+        patches = [
+            {"id": "sdk-app-startup", "disabled": True},
+            {"id": "sdk-jsonrpc-server", "disabled": True},
+            # Provider metadata is captured by verifiers, not uploaded by DSH.
+            {"id": "session-log-deepseek", "disabled": True},
+            {"id": "plugin-package-inventory-deepseek", "disabled": True},
+            {
+                "insert": [
+                    {
+                        "id": "acp-app-startup",
+                        "name": "@deepseek-ai/dsh-acp-app",
+                    },
+                    {
+                        "id": "acp",
+                        "name": "@deepseek-ai/dsh-acp",
+                        "inject": ["acpAppStartup"],
+                        "config": {"provider": "deepseek-official", "model": ctx.model},
+                    },
+                ]
+            },
+        ]
+        if system_prompt:
+            patches.append(
+                {
+                    "id": "system-prompt",
+                    "config": {
+                        "includeHarnessIdentity": False,
+                        "includeRuntimeContext": False,
+                        "personaPrefix": "You are a helpful software engineer assistant."
+                        + "\n\n"
+                        + system_prompt,
+                    },
+                }
+            )
+        patch_path = f"{home}/acp.patch.json"
+        # JSON is valid YAML and preserves arbitrary task prompt text verbatim.
+        await runtime.write(patch_path, json.dumps(patches).encode())
+        usage_path = f"{home}/usage.mjs"
+        await runtime.write(usage_path, USAGE_COMPAT)
+        return ACPConfig(
+            command=[
+                f"{NODE_BIN_DIR}/node",
+                "--import",
+                usage_path,
+                f"{DEEPSEEK_DIR.format(version=self.config.version)}/node_modules/@deepseek-ai/dsh/lib/bin.js",
+                "--profile",
+                "sdk-minimal",
+                "--patch",
+                patch_path,
+            ],
+            env={
+                **self.config.resolved_env,
+                "DSH_HOME": home,
+                "DEEPSEEK_BASE_URL": endpoint,
+                "DEEPSEEK_API_KEY": secret,
+            },
+            prompt=prompt,
+        )
+
+    async def gate_tools(
+        self, config: ACPConfig, runtime: Runtime, url: str, secret: str
+    ) -> None:
+        home = config.env["DSH_HOME"]
+        gate_path = f"{home}/gate.mjs"
+        await runtime.write(gate_path, GATE)
+        patch_path = f"{home}/acp.patch.json"
+        patches = json.loads(await runtime.read(patch_path))
+        patches.append(
+            {
+                "insert": [
+                    {"id": "vf-approval", "name": "@deepseek-ai/dsh-user-approval"},
+                    {"id": "vf-tool-gate", "name": gate_path},
+                ]
+            }
+        )
+        await runtime.write(patch_path, json.dumps(patches).encode())
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        await remove_dir(runtime, f"/tmp/vf-deepseek/{trace.id}", "DeepSeek state")

verifiers-0.4.1.dev32/verifiers/v1/harnesses/deepseek/usage.mjs

@@ -0,0 +1,75 @@
+// Some Messages-compatible gateways emit null for unknown cache counts. DSH's
+// parser accepts omitted counts, but rejects null. Keep the last known counts.
+const fetch = globalThis.fetch;
+const base = new URL(process.env.DEEPSEEK_BASE_URL);
+base.pathname = base.pathname.replace(/\/$/, "");
+if (!base.pathname.endsWith("/v1")) base.pathname += "/v1";
+base.pathname += "/messages";
+
+function normalize(line) {
+  if (!line.startsWith("data:")) return line;
+  let data;
+  try {
+    data = JSON.parse(line.slice(5));
+  } catch {
+    // Preserve SSE sentinels and non-JSON data for the upstream parser.
+    return line;
+  }
+  let changed = false;
+  for (const usage of [data?.usage, data?.message?.usage]) {
+    if (!usage) continue;
+    for (const key of ["cache_read_input_tokens", "cache_creation_input_tokens"]) {
+      if (usage[key] === null) {
+        delete usage[key];
+        changed = true;
+      }
+    }
+  }
+  return changed ? `data: ${JSON.stringify(data)}` : line;
+}
+
+globalThis.fetch = async (input, init) => {
+  const url = new URL(input instanceof Request ? input.url : input);
+  if (url.href === base.href && typeof init?.body === "string") {
+    const body = JSON.parse(init.body);
+    // DSH enables thinking without a budget; Messages gateways require either
+    // a budget or adaptive thinking. Eval sampling overrides apply downstream.
+    if (body.thinking?.type === "enabled" && body.thinking.budget_tokens === undefined) {
+      body.thinking = {type: "adaptive"};
+      init = {...init, body: JSON.stringify(body)};
+    }
+  }
+  const response = await fetch(input, init);
+  if (url.href === base.href && !response.ok) {
+    const detail = (await response.clone().text()).slice(0, 1000);
+    console.error(`DeepSeek Messages HTTP ${response.status}: ${detail}`);
+  }
+  if (
+    url.href !== base.href ||
+    !response.body ||
+    !response.headers.get("content-type")?.includes("text/event-stream")
+  ) return response;
+
+  let pending = "";
+  const body = response.body
+    .pipeThrough(new TextDecoderStream())
+    .pipeThrough(new TransformStream({
+      transform(chunk, controller) {
+        pending += chunk;
+        const lines = pending.split("\n");
+        pending = lines.pop();
+        for (const line of lines) controller.enqueue(`${normalize(line)}\n`);
+      },
+      flush(controller) {
+        if (pending) controller.enqueue(normalize(pending));
+      },
+    }))
+    .pipeThrough(new TextEncoderStream());
+  const headers = new Headers(response.headers);
+  headers.delete("content-length");
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  });
+};

{verifiers-0.4.1.dev30 → verifiers-0.4.1.dev32}/verifiers/v1/interception/server.py

@@ -288,7 +288,9 @@ async def _buffered_stream(
                 await stream.write(
                     replay.body
                     if response.status < 400
-                    else dialect.stream_error(from_json(replay.body))
+                    else dialect.stream_error(
+                        from_json(replay.body), status_code=response.status
+                    )
                 )
                 await stream.write_eof()
             except ConnectionResetError: