verifiers 0.4.1.dev31__tar.gz → 0.4.1.dev33__tar.gz

{verifiers-0.4.1.dev31 → verifiers-0.4.1.dev33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev31
+Version: 0.4.1.dev33
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev31 → verifiers-0.4.1.dev33}/pyproject.toml

@@ -237,6 +237,7 @@ markers = [
     "claude_code: integration cells on the claude-code harness",
     "openclaw: integration cells on the OpenClaw harness",
     "hermes_agent: integration cells on the hermes-agent harness",
+    "deepseek: integration cells on the DeepSeek minimal harness",
     "rlm: integration cells on the rlm harness",
     "kimi_code: integration cells on the kimi-code harness",
     "pi: integration cells on the pi harness",

{verifiers-0.4.1.dev31 → verifiers-0.4.1.dev33}/tests/integration/grid.py

@@ -49,6 +49,7 @@ HARNESSES: list[str | dict] = [
     "codex",
     "claude-code",
     "hermes-agent",
+    "deepseek",
     "kimi-code",
     "pi",
     "openclaw",

{verifiers-0.4.1.dev31 → verifiers-0.4.1.dev33}/verifiers/v1/harnesses/__init__.py

@@ -8,6 +8,7 @@ from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
+from verifiers.v1.harnesses.deepseek import DeepSeekHarness, DeepSeekHarnessConfig
 from verifiers.v1.harnesses.hermes_agent import (
     HermesAgentHarness,
     HermesAgentHarnessConfig,
@@ -37,6 +38,8 @@ __all__ = [
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
+    "DeepSeekHarness",
+    "DeepSeekHarnessConfig",
     "HermesAgentHarness",
     "HermesAgentHarnessConfig",
     "KimiCodeHarness",

verifiers-0.4.1.dev33/verifiers/v1/harnesses/deepseek/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.deepseek.harness import (
+    DeepSeekHarness,
+    DeepSeekHarnessConfig,
+)
+
+__all__ = ["DeepSeekHarness", "DeepSeekHarnessConfig"]

verifiers-0.4.1.dev33/verifiers/v1/harnesses/deepseek/gate.mjs

@@ -0,0 +1,11 @@
+// Preserve native policy denials; route every otherwise allowed tool through ACP.
+// The shared ACP runner resolves the call ID against the rollout's tool policy.
+export const name = "vf-tool-gate";
+export const inject = ["tools", "approval"];
+
+export function apply(ctx) {
+  ctx.on("tools/pre-execute", async (_exec, next) => {
+    const decision = await next();
+    return decision.kind === "allow" ? { kind: "ask" } : decision;
+  });
+}

verifiers-0.4.1.dev33/verifiers/v1/harnesses/deepseek/harness.py

@@ -0,0 +1,148 @@
+"""DeepSeek's minimal coding agent, driven through its native ACP bridge."""
+
+import json
+from pathlib import Path
+
+from verifiers.v1.acp import ACPConfig, ACPHarness
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
+from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
+from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
+from verifiers.v1.runtimes import Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+DEEPSEEK_DIR = "/var/tmp/vf-deepseek-{version}"
+USAGE_COMPAT = (Path(__file__).parent / "usage.mjs").read_bytes()
+GATE = (Path(__file__).parent / "gate.mjs").read_bytes()
+INSTALL = r"""
+set -e
+export PATH="/var/tmp/vf-node/bin:$PATH"
+npm install --prefix "$VF_DEEPSEEK_DIR" --no-audit --no-fund \
+    "@deepseek-ai/dsh@$VF_DEEPSEEK_VERSION" >/dev/null
+touch "$VF_DEEPSEEK_DIR/.ready"
+"""
+
+
+class DeepSeekHarnessConfig(HarnessConfig):
+    version: PinnedVersion = "0.2.1-alpha.3"
+    """DeepSeek CLI release, pinned to its shipped sdk-minimal composition."""
+
+
+class DeepSeekHarness(ACPHarness[DeepSeekHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_TOOL_INTERCEPTION = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await ensure_node(runtime)
+        directory = DEEPSEEK_DIR.format(version=self.config.version)
+        await ensure_installed(
+            runtime,
+            directory=directory,
+            ready=f"test -f {directory}/.ready",
+            install=INSTALL,
+            env={
+                "VF_DEEPSEEK_DIR": directory,
+                "VF_DEEPSEEK_VERSION": self.config.version,
+            },
+            label="DeepSeek Harness",
+        )
+        await super().setup(runtime)
+
+    async def prepare_acp(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ACPConfig:
+        if self.config.disabled_tools:
+            raise ValueError("DeepSeek minimal ACP does not support disabling tools")
+
+        home = f"/tmp/vf-deepseek/{trace.id}"
+        system_prompt, prompt = self.resolve_prompt(data)
+        # Keep the upstream minimal tree; replace only its SDK stdio front door.
+        # https://github.com/deepseek-ai/deepseek-harness/tree/master/packages/bundle/sdk-minimal
+        patches = [
+            {"id": "sdk-app-startup", "disabled": True},
+            {"id": "sdk-jsonrpc-server", "disabled": True},
+            # Provider metadata is captured by verifiers, not uploaded by DSH.
+            {"id": "session-log-deepseek", "disabled": True},
+            {"id": "plugin-package-inventory-deepseek", "disabled": True},
+            {
+                "insert": [
+                    {
+                        "id": "acp-app-startup",
+                        "name": "@deepseek-ai/dsh-acp-app",
+                    },
+                    {
+                        "id": "acp",
+                        "name": "@deepseek-ai/dsh-acp",
+                        "inject": ["acpAppStartup"],
+                        "config": {"provider": "deepseek-official", "model": ctx.model},
+                    },
+                ]
+            },
+        ]
+        if system_prompt:
+            patches.append(
+                {
+                    "id": "system-prompt",
+                    "config": {
+                        "includeHarnessIdentity": False,
+                        "includeRuntimeContext": False,
+                        "personaPrefix": "You are a helpful software engineer assistant."
+                        + "\n\n"
+                        + system_prompt,
+                    },
+                }
+            )
+        patch_path = f"{home}/acp.patch.json"
+        # JSON is valid YAML and preserves arbitrary task prompt text verbatim.
+        await runtime.write(patch_path, json.dumps(patches).encode())
+        usage_path = f"{home}/usage.mjs"
+        await runtime.write(usage_path, USAGE_COMPAT)
+        return ACPConfig(
+            command=[
+                f"{NODE_BIN_DIR}/node",
+                "--import",
+                usage_path,
+                f"{DEEPSEEK_DIR.format(version=self.config.version)}/node_modules/@deepseek-ai/dsh/lib/bin.js",
+                "--profile",
+                "sdk-minimal",
+                "--patch",
+                patch_path,
+            ],
+            env={
+                **self.config.resolved_env,
+                "DSH_HOME": home,
+                "DEEPSEEK_BASE_URL": endpoint,
+                "DEEPSEEK_API_KEY": secret,
+            },
+            prompt=prompt,
+        )
+
+    async def gate_tools(
+        self, config: ACPConfig, runtime: Runtime, url: str, secret: str
+    ) -> None:
+        home = config.env["DSH_HOME"]
+        gate_path = f"{home}/gate.mjs"
+        await runtime.write(gate_path, GATE)
+        patch_path = f"{home}/acp.patch.json"
+        patches = json.loads(await runtime.read(patch_path))
+        patches.append(
+            {
+                "insert": [
+                    {"id": "vf-approval", "name": "@deepseek-ai/dsh-user-approval"},
+                    {"id": "vf-tool-gate", "name": gate_path},
+                ]
+            }
+        )
+        await runtime.write(patch_path, json.dumps(patches).encode())
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        await remove_dir(runtime, f"/tmp/vf-deepseek/{trace.id}", "DeepSeek state")

verifiers-0.4.1.dev33/verifiers/v1/harnesses/deepseek/usage.mjs

@@ -0,0 +1,75 @@
+// Some Messages-compatible gateways emit null for unknown cache counts. DSH's
+// parser accepts omitted counts, but rejects null. Keep the last known counts.
+const fetch = globalThis.fetch;
+const base = new URL(process.env.DEEPSEEK_BASE_URL);
+base.pathname = base.pathname.replace(/\/$/, "");
+if (!base.pathname.endsWith("/v1")) base.pathname += "/v1";
+base.pathname += "/messages";
+
+function normalize(line) {
+  if (!line.startsWith("data:")) return line;
+  let data;
+  try {
+    data = JSON.parse(line.slice(5));
+  } catch {
+    // Preserve SSE sentinels and non-JSON data for the upstream parser.
+    return line;
+  }
+  let changed = false;
+  for (const usage of [data?.usage, data?.message?.usage]) {
+    if (!usage) continue;
+    for (const key of ["cache_read_input_tokens", "cache_creation_input_tokens"]) {
+      if (usage[key] === null) {
+        delete usage[key];
+        changed = true;
+      }
+    }
+  }
+  return changed ? `data: ${JSON.stringify(data)}` : line;
+}
+
+globalThis.fetch = async (input, init) => {
+  const url = new URL(input instanceof Request ? input.url : input);
+  if (url.href === base.href && typeof init?.body === "string") {
+    const body = JSON.parse(init.body);
+    // DSH enables thinking without a budget; Messages gateways require either
+    // a budget or adaptive thinking. Eval sampling overrides apply downstream.
+    if (body.thinking?.type === "enabled" && body.thinking.budget_tokens === undefined) {
+      body.thinking = {type: "adaptive"};
+      init = {...init, body: JSON.stringify(body)};
+    }
+  }
+  const response = await fetch(input, init);
+  if (url.href === base.href && !response.ok) {
+    const detail = (await response.clone().text()).slice(0, 1000);
+    console.error(`DeepSeek Messages HTTP ${response.status}: ${detail}`);
+  }
+  if (
+    url.href !== base.href ||
+    !response.body ||
+    !response.headers.get("content-type")?.includes("text/event-stream")
+  ) return response;
+
+  let pending = "";
+  const body = response.body
+    .pipeThrough(new TextDecoderStream())
+    .pipeThrough(new TransformStream({
+      transform(chunk, controller) {
+        pending += chunk;
+        const lines = pending.split("\n");
+        pending = lines.pop();
+        for (const line of lines) controller.enqueue(`${normalize(line)}\n`);
+      },
+      flush(controller) {
+        if (pending) controller.enqueue(normalize(pending));
+      },
+    }))
+    .pipeThrough(new TextEncoderStream());
+  const headers = new Headers(response.headers);
+  headers.delete("content-length");
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  });
+};

{verifiers-0.4.1.dev31 → verifiers-0.4.1.dev33}/verifiers/v1/judges/decision.py

@@ -21,7 +21,7 @@ from verifiers.v1.types import ID, Usage
 
 class DecisionJudgeConfig(JudgeConfig):
     id: ID = "decision"
-    model: str = "typesafe-ai/jev"
+    model: str = "cloudflare/clef"
     base_url: str = "https://api.pinference.ai/api/v1/evaluations"
     """Full URL to POST evaluation requests to."""
     levels: list[str] = Field(