verifiers 0.3.2.dev64__py3-none-any.whl → 0.3.2.dev66__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verifiers/v1/harnesses/bash/harness.py +0 -2
- verifiers/v1/harnesses/browser_use/program.py +53 -55
- verifiers/v1/harnesses/null/harness.py +0 -2
- verifiers/v1/harnesses/utils/core.py +31 -28
- verifiers/v1/harnesses/utils/launch.py +1 -4
- verifiers/v1/harnesses/utils/mcp.py +107 -33
- verifiers/v1/mcp/launch.py +28 -26
- verifiers/v1/runtimes/base.py +2 -0
- {verifiers-0.3.2.dev64.dist-info → verifiers-0.3.2.dev66.dist-info}/METADATA +1 -1
- {verifiers-0.3.2.dev64.dist-info → verifiers-0.3.2.dev66.dist-info}/RECORD +13 -13
- {verifiers-0.3.2.dev64.dist-info → verifiers-0.3.2.dev66.dist-info}/WHEEL +0 -0
- {verifiers-0.3.2.dev64.dist-info → verifiers-0.3.2.dev66.dist-info}/entry_points.txt +0 -0
- {verifiers-0.3.2.dev64.dist-info → verifiers-0.3.2.dev66.dist-info}/licenses/LICENSE +0 -0
|
@@ -8,7 +8,6 @@ from verifiers.v1.configs.harness import HarnessConfig
|
|
|
8
8
|
from verifiers.v1.harness import Harness
|
|
9
9
|
from verifiers.v1.harnesses.utils.launch import (
|
|
10
10
|
CHAT_PROGRAM_SOURCE,
|
|
11
|
-
MCP_CHAT_PROGRAM_SOURCE,
|
|
12
11
|
launch_chat_program,
|
|
13
12
|
)
|
|
14
13
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
@@ -122,7 +121,6 @@ class BashHarness(Harness[BashHarnessConfig]):
|
|
|
122
121
|
mcp_urls,
|
|
123
122
|
system_prompt,
|
|
124
123
|
prompt,
|
|
125
|
-
source_with_mcp=MCP_CHAT_PROGRAM_SOURCE,
|
|
126
124
|
extra_args=args,
|
|
127
125
|
env=env,
|
|
128
126
|
activate=False,
|
|
@@ -27,6 +27,7 @@ import subprocess
|
|
|
27
27
|
import sys
|
|
28
28
|
import time
|
|
29
29
|
import urllib.request
|
|
30
|
+
from contextlib import AsyncExitStack
|
|
30
31
|
from pathlib import Path
|
|
31
32
|
from typing import TYPE_CHECKING, Any, cast
|
|
32
33
|
from urllib.parse import urlsplit
|
|
@@ -226,63 +227,60 @@ async def main() -> None:
|
|
|
226
227
|
config = json.loads(args.mcp_config or "{}")
|
|
227
228
|
tools = [BROWSER_TOOL]
|
|
228
229
|
reserved = {"browser"}
|
|
229
|
-
|
|
230
|
-
await connect_mcp(config, reserved)
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
if
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
230
|
+
async with AsyncExitStack() as mcp_stack:
|
|
231
|
+
mcp_tools, dispatch, servers = await connect_mcp(config, mcp_stack, reserved)
|
|
232
|
+
tools += mcp_tools
|
|
233
|
+
messages = (
|
|
234
|
+
[{"role": "system", "content": args.system_prompt}]
|
|
235
|
+
if args.system_prompt
|
|
236
|
+
else []
|
|
237
|
+
)
|
|
238
|
+
if initial:
|
|
239
|
+
messages.extend(initial)
|
|
240
|
+
elif args.prompt:
|
|
241
|
+
messages.append({"role": "user", "content": args.prompt})
|
|
242
|
+
while True:
|
|
243
|
+
message = await chat(client, args.model, messages, tools)
|
|
244
|
+
messages.append(message.model_dump(exclude_none=True))
|
|
245
|
+
if not message.tool_calls:
|
|
246
|
+
break
|
|
247
|
+
for call in message.tool_calls:
|
|
248
|
+
name = call.function.name
|
|
249
|
+
try:
|
|
250
|
+
tool_args = json.loads(call.function.arguments or "{}")
|
|
251
|
+
except json.JSONDecodeError as e:
|
|
252
|
+
messages.append(
|
|
253
|
+
{
|
|
254
|
+
"role": "tool",
|
|
255
|
+
"tool_call_id": call.id,
|
|
256
|
+
"content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
|
|
257
|
+
}
|
|
258
|
+
)
|
|
259
|
+
continue
|
|
260
|
+
# Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
|
|
261
|
+
# below assume a dict, so reject anything else as a tool error rather than crashing.
|
|
262
|
+
if not isinstance(tool_args, dict):
|
|
263
|
+
messages.append(
|
|
264
|
+
{
|
|
265
|
+
"role": "tool",
|
|
266
|
+
"tool_call_id": call.id,
|
|
267
|
+
"content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
continue
|
|
271
|
+
if name in dispatch:
|
|
272
|
+
content = await call_mcp(servers, dispatch, name, tool_args)
|
|
273
|
+
elif name == "browser":
|
|
274
|
+
content = await asyncio.to_thread(
|
|
275
|
+
run_browser,
|
|
276
|
+
tool_args.get("code", ""),
|
|
277
|
+
browser_env,
|
|
278
|
+
)
|
|
279
|
+
else:
|
|
280
|
+
content = f"error: unknown tool {name!r}"
|
|
265
281
|
messages.append(
|
|
266
|
-
{
|
|
267
|
-
"role": "tool",
|
|
268
|
-
"tool_call_id": call.id,
|
|
269
|
-
"content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
|
|
270
|
-
}
|
|
271
|
-
)
|
|
272
|
-
continue
|
|
273
|
-
if name in dispatch:
|
|
274
|
-
content = await call_mcp(servers, dispatch, name, tool_args)
|
|
275
|
-
elif name == "browser":
|
|
276
|
-
content = await asyncio.to_thread(
|
|
277
|
-
run_browser,
|
|
278
|
-
tool_args.get("code", ""),
|
|
279
|
-
browser_env,
|
|
282
|
+
{"role": "tool", "tool_call_id": call.id, "content": content}
|
|
280
283
|
)
|
|
281
|
-
else:
|
|
282
|
-
content = f"error: unknown tool {name!r}"
|
|
283
|
-
messages.append(
|
|
284
|
-
{"role": "tool", "tool_call_id": call.id, "content": content}
|
|
285
|
-
)
|
|
286
284
|
|
|
287
285
|
|
|
288
286
|
if __name__ == "__main__":
|
|
@@ -3,7 +3,6 @@ from verifiers.v1.configs.harness import HarnessConfig
|
|
|
3
3
|
from verifiers.v1.harness import Harness
|
|
4
4
|
from verifiers.v1.harnesses.utils.launch import (
|
|
5
5
|
CHAT_PROGRAM_SOURCE,
|
|
6
|
-
MCP_CHAT_PROGRAM_SOURCE,
|
|
7
6
|
launch_chat_program,
|
|
8
7
|
)
|
|
9
8
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
@@ -47,5 +46,4 @@ class NullHarness(Harness[NullHarnessConfig]):
|
|
|
47
46
|
mcp_urls,
|
|
48
47
|
system_prompt,
|
|
49
48
|
prompt,
|
|
50
|
-
source_with_mcp=MCP_CHAT_PROGRAM_SOURCE,
|
|
51
49
|
)
|
|
@@ -4,6 +4,7 @@ import argparse
|
|
|
4
4
|
import asyncio
|
|
5
5
|
import json
|
|
6
6
|
import subprocess
|
|
7
|
+
from contextlib import AsyncExitStack
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import TYPE_CHECKING
|
|
9
10
|
|
|
@@ -361,35 +362,37 @@ async def main() -> None:
|
|
|
361
362
|
if args.search:
|
|
362
363
|
tools.append(SEARCH_TOOL)
|
|
363
364
|
reserved.add("search")
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
365
|
+
async with AsyncExitStack() as mcp_stack:
|
|
366
|
+
if config.get("mcpServers"):
|
|
367
|
+
mcp_tools, dispatch, servers = await asyncio.wait_for(
|
|
368
|
+
connect_mcp(config, mcp_stack, reserved),
|
|
369
|
+
timeout=None if args.bash else 60,
|
|
370
|
+
)
|
|
371
|
+
else:
|
|
372
|
+
mcp_tools, dispatch, servers = [], {}, {}
|
|
373
|
+
tools += mcp_tools
|
|
374
|
+
messages = (
|
|
375
|
+
[{"role": "system", "content": args.system_prompt}]
|
|
376
|
+
if args.system_prompt
|
|
377
|
+
else []
|
|
367
378
|
)
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
args
|
|
385
|
-
args.summarize_at_tokens,
|
|
386
|
-
)
|
|
387
|
-
if compactor.enabled and compactor.threshold is None:
|
|
388
|
-
compactor.threshold = await discover_threshold(client, args.model)
|
|
389
|
-
# The initial conversation is the floor for checkpoint fallbacks: a first-turn
|
|
390
|
-
# checkpoint must never retry from an empty base.
|
|
391
|
-
compactor.note_good(messages)
|
|
392
|
-
await run_chat_loop(args, compactor, messages, dispatch, servers, tool_client)
|
|
379
|
+
if initial:
|
|
380
|
+
messages.extend(initial)
|
|
381
|
+
elif args.prompt:
|
|
382
|
+
messages.append({"role": "user", "content": args.prompt})
|
|
383
|
+
compactor = Compactor(
|
|
384
|
+
client,
|
|
385
|
+
args.model,
|
|
386
|
+
tools,
|
|
387
|
+
args.compaction,
|
|
388
|
+
args.summarize_at_tokens,
|
|
389
|
+
)
|
|
390
|
+
if compactor.enabled and compactor.threshold is None:
|
|
391
|
+
compactor.threshold = await discover_threshold(client, args.model)
|
|
392
|
+
# The initial conversation is the floor for checkpoint fallbacks: a first-turn
|
|
393
|
+
# checkpoint must never retry from an empty base.
|
|
394
|
+
compactor.note_good(messages)
|
|
395
|
+
await run_chat_loop(args, compactor, messages, dispatch, servers, tool_client)
|
|
393
396
|
if tool_client is not None:
|
|
394
397
|
await tool_client.aclose()
|
|
395
398
|
|
|
@@ -29,8 +29,7 @@ CHAT_PROGRAM = (
|
|
|
29
29
|
'# dependencies = ["openai", "mcp==2.0.0", "httpx", "httpx2", "tenacity"]\n'
|
|
30
30
|
"# ///\n"
|
|
31
31
|
)
|
|
32
|
-
CHAT_PROGRAM_SOURCE = bundle_program(CHAT_PROGRAM, compaction, core)
|
|
33
|
-
MCP_CHAT_PROGRAM_SOURCE = bundle_program(CHAT_PROGRAM, mcp, compaction, core)
|
|
32
|
+
CHAT_PROGRAM_SOURCE = bundle_program(CHAT_PROGRAM, mcp, compaction, core)
|
|
34
33
|
|
|
35
34
|
|
|
36
35
|
async def launch_chat_program(
|
|
@@ -45,7 +44,6 @@ async def launch_chat_program(
|
|
|
45
44
|
system_prompt: str | None,
|
|
46
45
|
prompt: str | Messages | None,
|
|
47
46
|
*,
|
|
48
|
-
source_with_mcp: str | None = None,
|
|
49
47
|
extra_args: Sequence[str] = (),
|
|
50
48
|
env: dict[str, str] | None = None,
|
|
51
49
|
activate: bool = True,
|
|
@@ -60,7 +58,6 @@ async def launch_chat_program(
|
|
|
60
58
|
if system_prompt:
|
|
61
59
|
args.append(f"--system-prompt={system_prompt}")
|
|
62
60
|
if mcp_urls:
|
|
63
|
-
source = source_with_mcp or source
|
|
64
61
|
args.append(
|
|
65
62
|
"--mcp-config="
|
|
66
63
|
+ json.dumps(
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
|
2
3
|
from contextlib import AsyncExitStack, asynccontextmanager, suppress
|
|
3
|
-
from typing import Any, TypeVar, cast
|
|
4
|
+
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
|
4
5
|
|
|
5
|
-
import
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
from
|
|
6
|
+
from anyio import CancelScope
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from mcp import Client
|
|
9
10
|
|
|
10
11
|
MCP_CALL_ATTEMPTS = 6
|
|
11
12
|
MCP_TIMEOUT = 600.0
|
|
@@ -14,13 +15,21 @@ T = TypeVar("T")
|
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
@asynccontextmanager
|
|
17
|
-
async def mcp_client(spec: dict[str, Any]) -> AsyncIterator[Client]:
|
|
18
|
+
async def mcp_client(spec: dict[str, Any]) -> AsyncIterator["Client"]:
|
|
18
19
|
"""Open one fresh MCP client in the caller's task.
|
|
19
20
|
|
|
20
21
|
The client negotiates the newest protocol and falls back for older servers.
|
|
21
22
|
Teardown failures after the body completes are suppressed so closing noise cannot
|
|
22
23
|
fail or replay a call whose result is already available.
|
|
23
24
|
"""
|
|
25
|
+
# Bundled chat programs also run without tools; load MCP only when it is used.
|
|
26
|
+
import httpx2
|
|
27
|
+
from mcp import Client
|
|
28
|
+
from mcp.client.streamable_http import (
|
|
29
|
+
create_mcp_http_client,
|
|
30
|
+
streamable_http_client,
|
|
31
|
+
)
|
|
32
|
+
|
|
24
33
|
stack = AsyncExitStack()
|
|
25
34
|
try:
|
|
26
35
|
http_client = await stack.enter_async_context(
|
|
@@ -39,38 +48,105 @@ async def mcp_client(spec: dict[str, Any]) -> AsyncIterator[Client]:
|
|
|
39
48
|
await stack.aclose()
|
|
40
49
|
|
|
41
50
|
|
|
42
|
-
|
|
43
|
-
"""
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
class MCPConnection:
|
|
52
|
+
"""One rollout-owned client, with operations serialized across reconnects.
|
|
53
|
+
|
|
54
|
+
The SDK's AnyIO contexts enter and exit in the owner task, even when discovery
|
|
55
|
+
runs under wait_for or a tool call is cancelled by another task.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(self, spec: dict[str, Any]):
|
|
59
|
+
# Bundled programs without MCP should not import the retry machinery.
|
|
60
|
+
from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
|
|
61
|
+
|
|
62
|
+
self.spec = spec
|
|
63
|
+
self.task: asyncio.Task[None] | None = None
|
|
64
|
+
self.ready: asyncio.Future[Client] | None = None
|
|
65
|
+
self.cancel_scope: CancelScope | None = None
|
|
66
|
+
self.lock = asyncio.Lock()
|
|
67
|
+
self.run = AsyncRetrying(
|
|
68
|
+
stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
|
|
69
|
+
wait=wait_exponential_jitter(initial=0.5, max=30),
|
|
70
|
+
reraise=True,
|
|
71
|
+
).wraps(self._run)
|
|
72
|
+
|
|
73
|
+
async def serve(
|
|
74
|
+
self, ready: asyncio.Future["Client"], cancel_scope: CancelScope
|
|
75
|
+
) -> None:
|
|
76
|
+
stack = AsyncExitStack()
|
|
77
|
+
try:
|
|
78
|
+
stack.enter_context(cancel_scope)
|
|
79
|
+
client = await stack.enter_async_context(mcp_client(self.spec))
|
|
80
|
+
ready.set_result(client)
|
|
81
|
+
await asyncio.Future()
|
|
82
|
+
except asyncio.CancelledError:
|
|
83
|
+
pass
|
|
84
|
+
except Exception as error: # noqa: BLE001 - deliver owner failures to the caller
|
|
85
|
+
if not ready.done():
|
|
86
|
+
ready.set_exception(error)
|
|
87
|
+
finally:
|
|
88
|
+
if not ready.done():
|
|
89
|
+
ready.cancel()
|
|
90
|
+
# Owner teardown must not replace the caller's error with cancellation.
|
|
91
|
+
with suppress(asyncio.CancelledError):
|
|
92
|
+
await stack.aclose()
|
|
93
|
+
|
|
94
|
+
async def _run(self, operation: Callable[["Client"], Awaitable[T]]) -> T:
|
|
95
|
+
await self.lock.acquire()
|
|
96
|
+
try:
|
|
97
|
+
if self.task is None or self.task.done():
|
|
98
|
+
self.ready = asyncio.get_running_loop().create_future()
|
|
99
|
+
self.cancel_scope = CancelScope()
|
|
100
|
+
self.task = asyncio.create_task(
|
|
101
|
+
self.serve(self.ready, self.cancel_scope)
|
|
102
|
+
)
|
|
103
|
+
assert self.ready is not None
|
|
104
|
+
client = await self.ready
|
|
105
|
+
return await operation(client)
|
|
106
|
+
except BaseException as error:
|
|
107
|
+
await self.aclose(abort=isinstance(error, asyncio.CancelledError))
|
|
108
|
+
raise
|
|
109
|
+
finally:
|
|
110
|
+
self.lock.release()
|
|
111
|
+
|
|
112
|
+
async def aclose(self, *, abort: bool = False) -> None:
|
|
113
|
+
if self.task is None:
|
|
114
|
+
return
|
|
115
|
+
task, self.task = self.task, None
|
|
116
|
+
cancel_scope = self.cancel_scope
|
|
117
|
+
assert cancel_scope is not None
|
|
118
|
+
caller = asyncio.current_task()
|
|
119
|
+
assert caller is not None
|
|
120
|
+
# Stack cleanup can follow cancellation while awaiting a model or local tool.
|
|
121
|
+
if abort or caller.cancelling():
|
|
122
|
+
cancel_scope.cancel()
|
|
123
|
+
else:
|
|
124
|
+
task.cancel()
|
|
125
|
+
try:
|
|
126
|
+
await asyncio.shield(task)
|
|
127
|
+
except asyncio.CancelledError:
|
|
128
|
+
cancel_scope.cancel()
|
|
129
|
+
await asyncio.shield(task)
|
|
130
|
+
raise
|
|
52
131
|
|
|
53
132
|
|
|
54
133
|
async def connect_mcp(
|
|
55
|
-
config: dict[str, Any], reserved: set[str] | None = None
|
|
134
|
+
config: dict[str, Any], stack: AsyncExitStack, reserved: set[str] | None = None
|
|
56
135
|
) -> tuple[
|
|
57
136
|
list[dict[str, Any]],
|
|
58
137
|
dict[str, tuple[str, str]],
|
|
59
|
-
dict[str,
|
|
138
|
+
dict[str, MCPConnection],
|
|
60
139
|
]:
|
|
61
140
|
"""Enumerate MCP tools and return their schemas, dispatch map, and servers."""
|
|
62
141
|
tool_schemas: list[dict[str, Any]] = []
|
|
63
142
|
dispatch: dict[str, tuple[str, str]] = {}
|
|
64
|
-
servers: dict[str,
|
|
143
|
+
servers: dict[str, MCPConnection] = {}
|
|
65
144
|
reserved = reserved or set()
|
|
66
145
|
for name, spec in config.get("mcpServers", {}).items():
|
|
67
|
-
servers[name] = spec
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
return (await client.list_tools()).tools
|
|
72
|
-
|
|
73
|
-
for tool in await with_retry(list_tools):
|
|
146
|
+
server = servers[name] = MCPConnection(spec)
|
|
147
|
+
stack.push_async_callback(server.aclose)
|
|
148
|
+
result = await server.run(lambda client: client.list_tools())
|
|
149
|
+
for tool in result.tools:
|
|
74
150
|
full = f"{name}_{tool.name}" if name else tool.name
|
|
75
151
|
if full in reserved or full in dispatch:
|
|
76
152
|
raise ValueError(
|
|
@@ -111,17 +187,15 @@ def mcp_content_to_chat_content(
|
|
|
111
187
|
|
|
112
188
|
|
|
113
189
|
async def call_mcp(
|
|
114
|
-
servers: dict[str,
|
|
190
|
+
servers: dict[str, MCPConnection],
|
|
115
191
|
dispatch: dict[str, tuple[str, str]],
|
|
116
192
|
name: str,
|
|
117
193
|
arguments: dict[str, Any],
|
|
118
194
|
) -> str | list[dict[str, Any]]:
|
|
119
|
-
"""
|
|
195
|
+
"""Reuse the rollout's client, reconnecting before retrying a failed call."""
|
|
120
196
|
server_name, raw = dispatch[name]
|
|
121
197
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
result = await with_retry(call)
|
|
198
|
+
result = await servers[server_name].run(
|
|
199
|
+
lambda client: client.call_tool(raw, arguments)
|
|
200
|
+
)
|
|
127
201
|
return mcp_content_to_chat_content(result.content)
|
verifiers/v1/mcp/launch.py
CHANGED
|
@@ -32,6 +32,7 @@ from verifiers.v1.runtimes import (
|
|
|
32
32
|
)
|
|
33
33
|
from verifiers.v1.runtimes.base import _ENSURE_UV
|
|
34
34
|
from verifiers.v1.state import State
|
|
35
|
+
from verifiers.v1.utils.aio import run_shielded
|
|
35
36
|
|
|
36
37
|
if TYPE_CHECKING:
|
|
37
38
|
from verifiers.v1.mcp.toolset import Toolset
|
|
@@ -188,35 +189,36 @@ async def _install_in_sandbox(server: ServerBase, runtime: Runtime) -> str:
|
|
|
188
189
|
root = str(PurePosixPath(workdir) / ".vf-src")
|
|
189
190
|
temp = str(PurePosixPath(workdir) / ".vf-tmp")
|
|
190
191
|
cache = str(PurePosixPath(workdir) / ".vf-uv-cache")
|
|
191
|
-
vf, env = _verifiers_root(), Path(source_dir)
|
|
192
|
-
vf_name, vf_data = await _cached_sdist(vf)
|
|
193
|
-
if env == vf:
|
|
194
|
-
env_name, env_data = vf_name, vf_data
|
|
195
|
-
else:
|
|
196
|
-
env_name, env_data = await _cached_sdist(env)
|
|
197
|
-
vf_remote = f"{root}/{vf_name}"
|
|
198
|
-
env_remote = f"{root}/{env_name}"
|
|
199
|
-
await runtime.write(vf_remote, vf_data)
|
|
200
|
-
if env_remote != vf_remote:
|
|
201
|
-
await runtime.write(env_remote, env_data)
|
|
202
192
|
venv = str(PurePosixPath(workdir) / ".vf-venv")
|
|
203
193
|
root_q, temp_q, cache_q, venv_q = map(shlex.quote, (root, temp, cache, venv))
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
if result.exit_code != 0:
|
|
216
|
-
raise ToolsetError(
|
|
217
|
-
f"server {server.server_name!r} install failed in runtime: "
|
|
218
|
-
f"{(result.stderr or result.stdout).strip()[-2000:]}"
|
|
194
|
+
# Colocated servers and borrowed views install into one physical environment.
|
|
195
|
+
# Serialize its mutations and only remember sources after a successful install.
|
|
196
|
+
async with runtime._mcp_install_lock:
|
|
197
|
+
sources = dict.fromkeys((str(_verifiers_root()), source_dir))
|
|
198
|
+
pending = [source for source in sources if source not in runtime._mcp_sources]
|
|
199
|
+
if not pending:
|
|
200
|
+
return f"{venv}/bin/python"
|
|
201
|
+
setup = (
|
|
202
|
+
f"set -e; mkdir -p {root_q} {temp_q} {cache_q}; "
|
|
203
|
+
f"export TMPDIR={temp_q} UV_CACHE_DIR={cache_q}; "
|
|
204
|
+
'export PATH="$HOME/.local/bin:$PATH"; '
|
|
219
205
|
)
|
|
206
|
+
if not runtime._mcp_sources:
|
|
207
|
+
# Failed installs can leave the venv behind; retain it when retrying.
|
|
208
|
+
setup += f"{_ENSURE_UV}; uv venv --allow-existing {venv_q}; "
|
|
209
|
+
# Drain remote writes and installs before cancellation releases the lock.
|
|
210
|
+
for source in pending:
|
|
211
|
+
name, data = await _cached_sdist(Path(source))
|
|
212
|
+
remote = f"{root}/{name}"
|
|
213
|
+
await run_shielded(runtime.write(remote, data))
|
|
214
|
+
setup += f"uv pip install --python {venv_q} {shlex.quote(remote)}; "
|
|
215
|
+
result = await run_shielded(runtime.run(["sh", "-c", setup], {}))
|
|
216
|
+
if result.exit_code != 0:
|
|
217
|
+
raise ToolsetError(
|
|
218
|
+
f"server {server.server_name!r} install failed in runtime: "
|
|
219
|
+
f"{(result.stderr or result.stdout).strip()[-2000:]}"
|
|
220
|
+
)
|
|
221
|
+
runtime._mcp_sources.update(pending)
|
|
220
222
|
return f"{venv}/bin/python"
|
|
221
223
|
|
|
222
224
|
|
verifiers/v1/runtimes/base.py
CHANGED
|
@@ -155,6 +155,8 @@ class Runtime(ABC):
|
|
|
155
155
|
self.env: dict[str, str] = {}
|
|
156
156
|
self._uv_interpreters: dict[str, str] = {}
|
|
157
157
|
self._uv_script_locks: dict[str, asyncio.Lock] = {}
|
|
158
|
+
self._mcp_sources: set[str] = set()
|
|
159
|
+
self._mcp_install_lock = asyncio.Lock()
|
|
158
160
|
self._setup_claimed = False
|
|
159
161
|
self.stopped = False
|
|
160
162
|
"""Whether teardown has begun (set by `stop`). A stopped runtime is dead: a rollout
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.3.2.
|
|
3
|
+
Version: 0.3.2.dev66
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -85,10 +85,10 @@ verifiers/v1/gepa/runner.py,sha256=fGEyFzejdQKBzBbvdiTDHV3JtRhXMXMGSBU5y8C8i3w,6
|
|
|
85
85
|
verifiers/v1/harnesses/__init__.py,sha256=0FVi-mPF36fF0vpLY4H2vbK3U_om1Svz3KgJ3i-phs0,1918
|
|
86
86
|
verifiers/v1/harnesses/node.py,sha256=-lDPh30p65IETPVLkKn4yhEK49rtEGVQf9TNs7MVQYE,2133
|
|
87
87
|
verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
|
|
88
|
-
verifiers/v1/harnesses/bash/harness.py,sha256=
|
|
88
|
+
verifiers/v1/harnesses/bash/harness.py,sha256=nsjLKFMGFSya4s9fMHiNVrO92RRCTS3VHqTK3rv3AFA,5233
|
|
89
89
|
verifiers/v1/harnesses/browser_use/__init__.py,sha256=yxm2IG5TDLadxZEkX8SL26XWbcPcODXcrJfPfm1GqwU,171
|
|
90
90
|
verifiers/v1/harnesses/browser_use/harness.py,sha256=eGdLDRbTF8hg4NUwU1jVJcsUei0Y-gTqbGcj3eFJ7XI,4926
|
|
91
|
-
verifiers/v1/harnesses/browser_use/program.py,sha256=
|
|
91
|
+
verifiers/v1/harnesses/browser_use/program.py,sha256=5g5AnRS-brYSMmHcteLf8-XDLaTDNIOjG5dym91blOc,10051
|
|
92
92
|
verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umuahmOSqFjTrJ2hryTw,171
|
|
93
93
|
verifiers/v1/harnesses/claude_code/harness.py,sha256=Racxbs1EAH3gRwSeUptAIa3kB-EUm_nreUnHMwRUHuE,4119
|
|
94
94
|
verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
|
|
@@ -102,7 +102,7 @@ verifiers/v1/harnesses/mini_swe_agent/__init__.py,sha256=1OBu2Y-XqL72-dGhFdgB6sr
|
|
|
102
102
|
verifiers/v1/harnesses/mini_swe_agent/harness.py,sha256=HKGtlDyhXDJqNtDFeb3D4JYUerIgXK6HAmyqOm1mkRo,2473
|
|
103
103
|
verifiers/v1/harnesses/mini_swe_agent/program.py,sha256=Beej-JLIgSqnqIqB746oKU6kJYcynvppIbmwGa3nJJ0,167
|
|
104
104
|
verifiers/v1/harnesses/null/__init__.py,sha256=XDeKOPeoXUQi9US9_Z3_yH9YaZ4EiOMKJmUtNNdtD7s,127
|
|
105
|
-
verifiers/v1/harnesses/null/harness.py,sha256=
|
|
105
|
+
verifiers/v1/harnesses/null/harness.py,sha256=Bq19j8Bh_d4VvpFx9nwv6BKnG1tKweEk-2BSmaXWLkU,1330
|
|
106
106
|
verifiers/v1/harnesses/openclaw/__init__.py,sha256=kNyfPB1sGwEf4K3mTfKN0VEyxun3SGvNpvArVKvH-x8,160
|
|
107
107
|
verifiers/v1/harnesses/openclaw/harness.py,sha256=tucrQb_Osi7KAEtpyO4i55IXI5SFtLnyItB8eV4kjng,10534
|
|
108
108
|
verifiers/v1/harnesses/pi/__init__.py,sha256=1mSnzOf-RdEchdeg6IHtcdobG5KIpAhuTdUIgsCIn74,117
|
|
@@ -118,10 +118,10 @@ verifiers/v1/harnesses/terminus_2/harness.py,sha256=DMTY1bzvydyjHqzmvBSBGKJsmUzY
|
|
|
118
118
|
verifiers/v1/harnesses/terminus_2/program.py,sha256=X8MZdyItU3gbXGXnGOlJ9LWcUTnqIDaxlhkZvoRiqKY,2703
|
|
119
119
|
verifiers/v1/harnesses/utils/__init__.py,sha256=eTS-Ft2HwoLiYn5u1t_KueLmaZB_W12cdYmcVsE-G5A,364
|
|
120
120
|
verifiers/v1/harnesses/utils/compaction.py,sha256=DeY6CYLEsv7-vJm26sT2mRaPHu3W2ywiZhvFr_pi6Hg,10495
|
|
121
|
-
verifiers/v1/harnesses/utils/core.py,sha256=
|
|
121
|
+
verifiers/v1/harnesses/utils/core.py,sha256=x3znrq2CkXmdXRCOtamhHN86fI5EaD6zr7krV6n0BnI,15221
|
|
122
122
|
verifiers/v1/harnesses/utils/install.py,sha256=BDkkUVZ1brSzCrbQOvMO5mVIE2DsV2pE7rTC5yROYUw,2992
|
|
123
|
-
verifiers/v1/harnesses/utils/launch.py,sha256=
|
|
124
|
-
verifiers/v1/harnesses/utils/mcp.py,sha256=
|
|
123
|
+
verifiers/v1/harnesses/utils/launch.py,sha256=RqPuUY-kT2XVuLSGtXW7kRQEaP-o2KzkiJ7vcoo2neM,2883
|
|
124
|
+
verifiers/v1/harnesses/utils/mcp.py,sha256=oNhKfLjAW-wScJcDIsW_HfuvKY-JkaGBBZrRLqgRK1g,7355
|
|
125
125
|
verifiers/v1/interception/__init__.py,sha256=5Kvo7tcBUVaa65s9A1NiOH3CPpzw-P5btXwxB4Nbbvs,4251
|
|
126
126
|
verifiers/v1/interception/base.py,sha256=Mg16Y104lyZilF9shdivdqYoqz4OYp978i_myyrulu8,2529
|
|
127
127
|
verifiers/v1/interception/pool.py,sha256=sk_zcdTV8EVRuzqlYMXxon_42tSDAlFvwcyOO2EF7hw,6473
|
|
@@ -137,11 +137,11 @@ verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93
|
|
|
137
137
|
verifiers/v1/judges/rubric.py,sha256=HGQYI3xdq8I84Ce0_sDeNgk5JM597es8rOVPipjCyxU,12025
|
|
138
138
|
verifiers/v1/judges/rubric.txt,sha256=3KA2ZeYYdcDweJhUGopxu9EFgARhu2aWft5S5cj8Bc8,323
|
|
139
139
|
verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,408
|
|
140
|
-
verifiers/v1/mcp/launch.py,sha256=
|
|
140
|
+
verifiers/v1/mcp/launch.py,sha256=gFkAdSZpFAxj5AAmL45Rh1dAttZVzoXJQPqgfY-jcGU,23866
|
|
141
141
|
verifiers/v1/mcp/server.py,sha256=Jqi8OMT8DnoZegXqM3UNNRzhVnbdrPEwx0DjOJo_K2A,13256
|
|
142
142
|
verifiers/v1/mcp/toolset.py,sha256=G2wzNdEMGLXoQUcrwF0TartJkqokNPI3o7YUOAaaa6U,1007
|
|
143
143
|
verifiers/v1/runtimes/__init__.py,sha256=ppc1R_sx-aMcuJSx4t99rTSjloUXI0M6EUj676-1PeY,2862
|
|
144
|
-
verifiers/v1/runtimes/base.py,sha256=
|
|
144
|
+
verifiers/v1/runtimes/base.py,sha256=VMxYMnGE_zj4k1VseTxEtKhzOZroahRnZSfvruDenEw,17468
|
|
145
145
|
verifiers/v1/runtimes/limiters.py,sha256=C6cWDD4pwyFwUY4QFke3oiMXwydSNIfSbJSIglUC2kE,3062
|
|
146
146
|
verifiers/v1/runtimes/modal.py,sha256=wUNik2Wul1YISausdABIMAoSaTA0MNDwKUayKxVLwAA,13342
|
|
147
147
|
verifiers/v1/runtimes/prime.py,sha256=fbyR5s3WqsJNWki3eHJa83j4iobnqDugdJWm8GwEs7w,18494
|
|
@@ -189,8 +189,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
|
|
|
189
189
|
verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
|
|
190
190
|
verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
|
|
191
191
|
verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
|
|
192
|
-
verifiers-0.3.2.
|
|
193
|
-
verifiers-0.3.2.
|
|
194
|
-
verifiers-0.3.2.
|
|
195
|
-
verifiers-0.3.2.
|
|
196
|
-
verifiers-0.3.2.
|
|
192
|
+
verifiers-0.3.2.dev66.dist-info/METADATA,sha256=_cPGErluCxAPydFeyGvLRsw5QHmMadVtWvNOrud1fG0,4204
|
|
193
|
+
verifiers-0.3.2.dev66.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
194
|
+
verifiers-0.3.2.dev66.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
|
|
195
|
+
verifiers-0.3.2.dev66.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
|
|
196
|
+
verifiers-0.3.2.dev66.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|