verifiers 0.2.1.dev63__tar.gz → 0.2.1.dev64__tar.gz

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev63
+Version: 0.2.1.dev64
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev64}/verifiers/v1/harnesses/__init__.py

@@ -11,6 +11,7 @@ from verifiers.v1.harnesses.mini_swe_agent import (
 )
 from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.pi import PiHarness, PiHarnessConfig
+from verifiers.v1.harnesses.pool import PoolHarness, PoolHarnessConfig
 from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
@@ -27,6 +28,8 @@ __all__ = [
     "NullHarnessConfig",
     "PiHarness",
     "PiHarnessConfig",
+    "PoolHarness",
+    "PoolHarnessConfig",
     "MiniSWEAgentHarness",
     "MiniSWEAgentHarnessConfig",
     "RLMHarness",

verifiers-0.2.1.dev64/verifiers/v1/harnesses/pool/__init__.py

@@ -0,0 +1,3 @@
+from verifiers.v1.harnesses.pool.harness import PoolHarness, PoolHarnessConfig
+
+__all__ = ["PoolHarness", "PoolHarnessConfig"]

verifiers-0.2.1.dev64/verifiers/v1/harnesses/pool/harness.py

@@ -0,0 +1,130 @@
+"""Run Poolside's `pool exec` against interception as an OpenAI-compatible provider."""
+
+import json
+import shlex
+
+from pydantic import Field
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import SystemMessage, TextContentPart, UserMessage
+
+POOL_DIR = "/tmp/vf-pool-{version}"
+SETTINGS_PATH = ".poolside/settings.local.yaml"
+INSTALL = r"""
+set -e
+command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
+command -v tar >/dev/null || (apt-get update -qq && apt-get install -y -qq tar >/dev/null)
+case "$(uname -s)" in Linux) os=linux ;; Darwin) os=darwin ;; *) echo "unsupported os: $(uname -s)" >&2; exit 1 ;; esac
+case "$(uname -m)" in aarch64|arm64) arch=arm64 ;; x86_64|amd64) arch=amd64 ;; *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;; esac
+mkdir -p {dir}
+curl -fsSL "https://github.com/poolsideai/pool/releases/download/v{version}/pool-$os-$arch.tar.gz" | tar -xz -C {dir}
+mv "{dir}/pool-$os-$arch" "{dir}/pool"
+chmod +x "{dir}/pool"
+"""
+
+
+class PoolHarnessConfig(HarnessConfig):
+    version: str = Field(default="1.0.11", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Pool release to install, pinned for reproducibility."""
+
+
+class PoolHarness(Harness[PoolHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_MESSAGE_PROMPT = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        directory = POOL_DIR.format(version=self.config.version)
+        binary = f"{directory}/pool"
+        script = INSTALL.replace("{version}", self.config.version).replace(
+            "{dir}", directory
+        )
+        ensure = shlex.quote(f"[ -x {binary} ] || ({script})")
+        # Cache the pinned binary across local rollouts; Linux has flock, macOS has lockf.
+        guarded = (
+            f"mkdir -p {directory} && "
+            f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
+            f"bash -o pipefail -c {ensure}"
+        )
+        result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env)
+        if result.exit_code != 0:
+            detail = (result.stderr or result.stdout).strip()[-500:]
+            raise RuntimeError(f"Pool install failed: {detail}")
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(trace.task.data)
+        if prompt is None:
+            raise ValueError("Pool requires a task prompt (it has no user simulator)")
+        texts = [system_prompt] if system_prompt else []
+        if isinstance(prompt, str):
+            texts.append(prompt)
+        else:
+            for message in prompt:
+                if not isinstance(message, (SystemMessage, UserMessage)):
+                    raise ValueError(
+                        "pool exec only supports system and user initial messages"
+                    )
+                parts = (
+                    [TextContentPart(text=message.content)]
+                    if isinstance(message.content, str)
+                    else message.content
+                )
+                for part in parts:
+                    if not isinstance(part, TextContentPart):
+                        raise ValueError("pool exec does not support image prompts")
+                    texts.append(part.text)
+        text = "\n\n".join(text for text in texts if text)
+
+        settings = {
+            "mcp_servers": {
+                name: {"transport": {"type": "http", "url": url}}
+                for name, url in mcp_urls.items()
+            },
+            # Values are Pool tool names such as `shell`, `read`, or `edit`.
+            "tools": {
+                name: {"disabled": True} for name in self.config.disabled_tools or []
+            },
+        }
+        await runtime.write(SETTINGS_PATH, json.dumps(settings).encode())
+
+        prompt_path = f".vf-pool/prompt-{trace.id}.txt"
+        await runtime.write(prompt_path, text.encode())
+        env = {
+            **self.config.resolved_env,
+            # Standalone provider mode sends this bearer and model to interception.
+            "POOLSIDE_API_KEY": secret,
+            "POOLSIDE_STANDALONE_BASE_URL": endpoint,
+            "POOLSIDE_STANDALONE_MODEL": ctx.model,
+        }
+        directory = POOL_DIR.format(version=self.config.version)
+        argv = [
+            f"{directory}/pool",
+            "exec",
+            "--unsafe-auto-allow",
+            "--sandbox",
+            "disabled",
+            "--prompt-file",
+            prompt_path,
+        ]
+        # Keep Pool's home, config, logs, and trajectories inside the rollout workspace.
+        isolate = (
+            'export HOME="$PWD/.vf-pool/home" '
+            'XDG_CONFIG_HOME="$PWD/.vf-pool/config" '
+            'XDG_STATE_HOME="$PWD/.vf-pool/state"; exec "$@"'
+        )
+        result = await runtime.run_program(["sh", "-c", isolate, "pool", *argv], env)
+        # Exit 4 means the agent declined the task, not that the harness failed.
+        if result.exit_code == 4:
+            return ProgramResult(0, result.stdout, result.stderr)
+        return result

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev64}/verifiers/v1/interception/server.py

@@ -626,7 +626,8 @@ class InterceptionServer(Interception):
                         async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS):
                             await ready.wait()
                     except TimeoutError:
-                        await resp.write(b": keepalive\n\n")
+                        # Don't terminate an empty event; some SSE clients try to JSON-decode it.
+                        await resp.write(b": keepalive\n")
                         continue
                     chunk = queue.get_nowait()
                     if queue.empty():
@@ -634,6 +635,13 @@ class InterceptionServer(Interception):
                     if chunk is None:
                         await producer
                         break
+                    # We send our own keepalive above. Some clients treat a complete
+                    # comment-only event from upstream as an empty JSON payload.
+                    if not any(
+                        line.startswith(b"data:") for line in chunk.splitlines()
+                    ):
+                        await resp.write(b": keepalive\n")
+                        continue
                     if deferred or dialect.is_terminal_event(chunk):
                         if parser_error is None:
                             try: