verifiers 0.2.1.dev63__tar.gz → 0.2.1.dev65__tar.gz

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev63
+Version: 0.2.1.dev65
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -24,33 +24,32 @@ Requires-Python: <3.14,>=3.11
 Requires-Dist: aiohttp>=3.9.0
 Requires-Dist: aiolimiter>=1.2.1
 Requires-Dist: anthropic>=0.78.0
-Requires-Dist: datasets<4.7.0,>=3.0.0
-Requires-Dist: gepa
+Requires-Dist: datasets<6.0.0,>=3.3.0
+Requires-Dist: gepa>=0.0.6
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: math-verify>=0.8.0
-Requires-Dist: mcp>=1.14.1
+Requires-Dist: mcp<2,>=1.24.0
 Requires-Dist: msgpack>=1.1.2
-Requires-Dist: numpy
-Requires-Dist: openai-agents>=0.0.7
-Requires-Dist: openai>=1.108.1
+Requires-Dist: numpy>=2.1.0
+Requires-Dist: openai-agents>=0.8.2
+Requires-Dist: openai>=2.9.0
 Requires-Dist: prime-pydantic-config[toml]>=0.3.0.dev86
 Requires-Dist: prime-sandboxes>=0.2.31
 Requires-Dist: prime-tunnel>=0.1.8
-Requires-Dist: pydantic>=2.11.9
+Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
 Requires-Dist: renderers>=0.1.8
 Requires-Dist: requests
-Requires-Dist: rich
+Requires-Dist: rich>=11.0.0
 Requires-Dist: setproctitle>=1.3.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0
-Requires-Dist: typing-extensions; python_version < '3.12'
+Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'
 Provides-Extra: browser
-Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
-Requires-Dist: stagehand>=3.0.0; extra == 'browser'
+Requires-Dist: stagehand>=3.4.0; extra == 'browser'
 Provides-Extra: harbor
 Requires-Dist: harbor==0.14.0; (python_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
@@ -59,23 +58,8 @@ Provides-Extra: notebook
 Requires-Dist: nest-asyncio>=1.6.0; extra == 'notebook'
 Provides-Extra: openenv
 Requires-Dist: openenv>=0.4.1; extra == 'openenv'
-Provides-Extra: quest
-Requires-Dist: certifi; extra == 'quest'
-Requires-Dist: pillow; extra == 'quest'
-Requires-Dist: pymupdf; extra == 'quest'
 Provides-Extra: rg
-Requires-Dist: reasoning-gym; extra == 'rg'
-Provides-Extra: rl
-Requires-Dist: accelerate>=1.4.0; extra == 'rl'
-Requires-Dist: deepspeed>=0.17.6; extra == 'rl'
-Requires-Dist: flash-attn>=2.8.3; extra == 'rl'
-Requires-Dist: liger-kernel>=0.5.10; extra == 'rl'
-Requires-Dist: peft; extra == 'rl'
-Requires-Dist: requests; extra == 'rl'
-Requires-Dist: torch<2.9.0,>=2.8.0; extra == 'rl'
-Requires-Dist: transformers>=4.56.2; extra == 'rl'
-Requires-Dist: vllm<0.11.0,>=0.10.0; extra == 'rl'
-Requires-Dist: wandb; extra == 'rl'
+Requires-Dist: reasoning-gym>=0.1.8; extra == 'rg'
 Provides-Extra: ta
 Requires-Dist: nltk>=3.9.2; extra == 'ta'
 Requires-Dist: textarena==0.7.4; extra == 'ta'

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev65}/pyproject.toml

@@ -28,19 +28,19 @@ classifiers = [
 
 dependencies = [
     "anthropic>=0.78.0",
-    "datasets>=3.0.0, <4.7.0",
-    "numpy",
+    "datasets>=3.3.0,<6.0.0",
+    "numpy>=2.1.0",
     "math-verify>=0.8.0",
-    "mcp>=1.14.1",
-    "openai>=1.108.1",
-    "openai-agents>=0.0.7",
+    "mcp>=1.24.0,<2",
+    "openai>=2.9.0",
+    "openai-agents>=0.8.2",
     "prime-tunnel>=0.1.8",
     "prime-sandboxes>=0.2.31",
-    "pydantic>=2.11.9",
+    "pydantic>=2.12.3",
     "requests",
-    "rich",
+    "rich>=11.0.0",
     "tenacity>=8.5.0",
-    "gepa",
+    "gepa>=0.0.6",
     "pyzmq>=27.1.0",
     "msgpack>=1.1.2",
     "aiolimiter>=1.2.1",
@@ -52,23 +52,21 @@ dependencies = [
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
     "renderers>=0.1.8",
-    "typing_extensions; python_version < '3.12'",
+    "typing_extensions>=4.12.2",
 ]
 
 [dependency-groups]
 dev = [
-    "ruff",
-    "pre-commit",
+    "ruff>=0.9.0",
+    "pre-commit>=2.19.0",
     "ty>=0.0.1a29,<0.0.22",
-    "pytest>=7.0.0",
+    "pytest>=7.0.0,<9.1.0",
     "pytest-asyncio>=0.21.0",
     "pytest-cov>=4.0.0",
     "pytest-xdist>=3.8.0",
-    "ipykernel",
-    "ipywidgets",
-    "reasoning-gym",
-    "stagehand>=3.0.0",
-    "aiohttp>=3.9.0",
+    "nest-asyncio>=1.6.0",
+    "reasoning-gym>=0.1.8",
+    "stagehand>=3.4.0",
     "python-dotenv>=1.0.0",
     "nltk>=3.9.2",
     "textarena==0.7.4",
@@ -85,15 +83,14 @@ examples = [
 
 [project.optional-dependencies]
 rg = [
-    "reasoning-gym",
+    "reasoning-gym>=0.1.8",
 ]
 ta = [
     "textarena==0.7.4",
     "nltk>=3.9.2",
 ]
 browser = [
-    "stagehand>=3.0.0",
-    "aiohttp>=3.9.0",
+    "stagehand>=3.4.0",
     "python-dotenv>=1.0.0",
 ]
 harbor = [
@@ -109,24 +106,6 @@ notebook = [
 openenv = [
     "openenv>=0.4.1",
 ]
-quest = [
-    "certifi",
-    "pillow",
-    "pymupdf",
-]
-rl = [
-    "torch>=2.8.0,<2.9.0",
-    "transformers>=4.56.2",
-    "accelerate>=1.4.0",
-    "requests",
-    "peft",
-    "wandb",
-    "vllm>=0.10.0,<0.11.0",
-    "liger-kernel>=0.5.10",
-    "deepspeed>=0.17.6",
-    "flash-attn>=2.8.3",
-]
-
 [tool.uv]
 preview = true
 required-version = ">=0.11.1"
@@ -159,12 +138,6 @@ prime-sandboxes = false
 prime-pydantic-config = false
 renderers = false
 
-[tool.uv.extra-build-dependencies]
-flash-attn = [{ requirement = "torch", match-runtime = true }]
-
-[tool.uv.extra-build-variables]
-flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
-
 [project.scripts]
 vf-eval = "verifiers.scripts.eval:main"
 eval = "verifiers.v1.cli.eval.main:main"

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev65}/verifiers/v1/harnesses/__init__.py

@@ -11,6 +11,7 @@ from verifiers.v1.harnesses.mini_swe_agent import (
 )
 from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.pi import PiHarness, PiHarnessConfig
+from verifiers.v1.harnesses.pool import PoolHarness, PoolHarnessConfig
 from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
@@ -27,6 +28,8 @@ __all__ = [
     "NullHarnessConfig",
     "PiHarness",
     "PiHarnessConfig",
+    "PoolHarness",
+    "PoolHarnessConfig",
     "MiniSWEAgentHarness",
     "MiniSWEAgentHarnessConfig",
     "RLMHarness",

verifiers-0.2.1.dev65/verifiers/v1/harnesses/pool/__init__.py

@@ -0,0 +1,3 @@
+from verifiers.v1.harnesses.pool.harness import PoolHarness, PoolHarnessConfig
+
+__all__ = ["PoolHarness", "PoolHarnessConfig"]

verifiers-0.2.1.dev65/verifiers/v1/harnesses/pool/harness.py

@@ -0,0 +1,130 @@
+"""Run Poolside's `pool exec` against interception as an OpenAI-compatible provider."""
+
+import json
+import shlex
+
+from pydantic import Field
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import SystemMessage, TextContentPart, UserMessage
+
+POOL_DIR = "/tmp/vf-pool-{version}"
+SETTINGS_PATH = ".poolside/settings.local.yaml"
+INSTALL = r"""
+set -e
+command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
+command -v tar >/dev/null || (apt-get update -qq && apt-get install -y -qq tar >/dev/null)
+case "$(uname -s)" in Linux) os=linux ;; Darwin) os=darwin ;; *) echo "unsupported os: $(uname -s)" >&2; exit 1 ;; esac
+case "$(uname -m)" in aarch64|arm64) arch=arm64 ;; x86_64|amd64) arch=amd64 ;; *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;; esac
+mkdir -p {dir}
+curl -fsSL "https://github.com/poolsideai/pool/releases/download/v{version}/pool-$os-$arch.tar.gz" | tar -xz -C {dir}
+mv "{dir}/pool-$os-$arch" "{dir}/pool"
+chmod +x "{dir}/pool"
+"""
+
+
+class PoolHarnessConfig(HarnessConfig):
+    version: str = Field(default="1.0.11", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Pool release to install, pinned for reproducibility."""
+
+
+class PoolHarness(Harness[PoolHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_MESSAGE_PROMPT = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        directory = POOL_DIR.format(version=self.config.version)
+        binary = f"{directory}/pool"
+        script = INSTALL.replace("{version}", self.config.version).replace(
+            "{dir}", directory
+        )
+        ensure = shlex.quote(f"[ -x {binary} ] || ({script})")
+        # Cache the pinned binary across local rollouts; Linux has flock, macOS has lockf.
+        guarded = (
+            f"mkdir -p {directory} && "
+            f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
+            f"bash -o pipefail -c {ensure}"
+        )
+        result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env)
+        if result.exit_code != 0:
+            detail = (result.stderr or result.stdout).strip()[-500:]
+            raise RuntimeError(f"Pool install failed: {detail}")
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(trace.task.data)
+        if prompt is None:
+            raise ValueError("Pool requires a task prompt (it has no user simulator)")
+        texts = [system_prompt] if system_prompt else []
+        if isinstance(prompt, str):
+            texts.append(prompt)
+        else:
+            for message in prompt:
+                if not isinstance(message, (SystemMessage, UserMessage)):
+                    raise ValueError(
+                        "pool exec only supports system and user initial messages"
+                    )
+                parts = (
+                    [TextContentPart(text=message.content)]
+                    if isinstance(message.content, str)
+                    else message.content
+                )
+                for part in parts:
+                    if not isinstance(part, TextContentPart):
+                        raise ValueError("pool exec does not support image prompts")
+                    texts.append(part.text)
+        text = "\n\n".join(text for text in texts if text)
+
+        settings = {
+            "mcp_servers": {
+                name: {"transport": {"type": "http", "url": url}}
+                for name, url in mcp_urls.items()
+            },
+            # Values are Pool tool names such as `shell`, `read`, or `edit`.
+            "tools": {
+                name: {"disabled": True} for name in self.config.disabled_tools or []
+            },
+        }
+        await runtime.write(SETTINGS_PATH, json.dumps(settings).encode())
+
+        prompt_path = f".vf-pool/prompt-{trace.id}.txt"
+        await runtime.write(prompt_path, text.encode())
+        env = {
+            **self.config.resolved_env,
+            # Standalone provider mode sends this bearer and model to interception.
+            "POOLSIDE_API_KEY": secret,
+            "POOLSIDE_STANDALONE_BASE_URL": endpoint,
+            "POOLSIDE_STANDALONE_MODEL": ctx.model,
+        }
+        directory = POOL_DIR.format(version=self.config.version)
+        argv = [
+            f"{directory}/pool",
+            "exec",
+            "--unsafe-auto-allow",
+            "--sandbox",
+            "disabled",
+            "--prompt-file",
+            prompt_path,
+        ]
+        # Keep Pool's home, config, logs, and trajectories inside the rollout workspace.
+        isolate = (
+            'export HOME="$PWD/.vf-pool/home" '
+            'XDG_CONFIG_HOME="$PWD/.vf-pool/config" '
+            'XDG_STATE_HOME="$PWD/.vf-pool/state"; exec "$@"'
+        )
+        result = await runtime.run_program(["sh", "-c", isolate, "pool", *argv], env)
+        # Exit 4 means the agent declined the task, not that the harness failed.
+        if result.exit_code == 4:
+            return ProgramResult(0, result.stdout, result.stderr)
+        return result

{verifiers-0.2.1.dev63 → verifiers-0.2.1.dev65}/verifiers/v1/interception/server.py

@@ -626,7 +626,8 @@ class InterceptionServer(Interception):
                         async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS):
                             await ready.wait()
                     except TimeoutError:
-                        await resp.write(b": keepalive\n\n")
+                        # Don't terminate an empty event; some SSE clients try to JSON-decode it.
+                        await resp.write(b": keepalive\n")
                         continue
                     chunk = queue.get_nowait()
                     if queue.empty():
@@ -634,6 +635,13 @@ class InterceptionServer(Interception):
                     if chunk is None:
                         await producer
                         break
+                    # We send our own keepalive above. Some clients treat a complete
+                    # comment-only event from upstream as an empty JSON payload.
+                    if not any(
+                        line.startswith(b"data:") for line in chunk.splitlines()
+                    ):
+                        await resp.write(b": keepalive\n")
+                        continue
                     if deferred or dialect.is_terminal_event(chunk):
                         if parser_error is None:
                             try: