verifiers 0.2.2.dev89__py3-none-any.whl → 0.2.2.dev91__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verifiers/v1/acp/__init__.py +9 -0
- verifiers/v1/acp/runner.py +22 -6
- verifiers/v1/harnesses/claude_code/harness.py +121 -61
- verifiers/v1/harnesses/codex/harness.py +152 -238
- verifiers/v1/harnesses/node.py +54 -0
- verifiers/v1/harnesses/pi/harness.py +6 -31
- {verifiers-0.2.2.dev89.dist-info → verifiers-0.2.2.dev91.dist-info}/METADATA +1 -1
- {verifiers-0.2.2.dev89.dist-info → verifiers-0.2.2.dev91.dist-info}/RECORD +11 -10
- {verifiers-0.2.2.dev89.dist-info → verifiers-0.2.2.dev91.dist-info}/WHEEL +0 -0
- {verifiers-0.2.2.dev89.dist-info → verifiers-0.2.2.dev91.dist-info}/entry_points.txt +0 -0
- {verifiers-0.2.2.dev89.dist-info → verifiers-0.2.2.dev91.dist-info}/licenses/LICENSE +0 -0
verifiers/v1/acp/__init__.py
CHANGED
|
@@ -46,6 +46,7 @@ class ACP:
|
|
|
46
46
|
command: list[str],
|
|
47
47
|
prompt: str | Messages | None,
|
|
48
48
|
system_prompt: str | None = None,
|
|
49
|
+
session_meta: dict | None = None,
|
|
49
50
|
) -> "ACPHarnessSession":
|
|
50
51
|
"""Create a persistent ACP-backed handle owned by one rollout."""
|
|
51
52
|
return ACPHarnessSession(
|
|
@@ -61,6 +62,7 @@ class ACP:
|
|
|
61
62
|
command=command,
|
|
62
63
|
prompt=prompt,
|
|
63
64
|
system_prompt=system_prompt,
|
|
65
|
+
session_meta=session_meta,
|
|
64
66
|
)
|
|
65
67
|
|
|
66
68
|
async def run(
|
|
@@ -73,6 +75,7 @@ class ACP:
|
|
|
73
75
|
mcp_urls: dict[str, str] | None = None,
|
|
74
76
|
system_prompt: str | None = None,
|
|
75
77
|
session_path: str | None = None,
|
|
78
|
+
session_meta: dict | None = None,
|
|
76
79
|
allow_empty_tool_reply: bool = False,
|
|
77
80
|
) -> ProgramResult:
|
|
78
81
|
"""Run one ACP segment without retaining its process."""
|
|
@@ -84,6 +87,7 @@ class ACP:
|
|
|
84
87
|
mcp_urls=mcp_urls,
|
|
85
88
|
system_prompt=system_prompt,
|
|
86
89
|
session_path=session_path,
|
|
90
|
+
session_meta=session_meta,
|
|
87
91
|
allow_empty_tool_reply=allow_empty_tool_reply,
|
|
88
92
|
)
|
|
89
93
|
|
|
@@ -97,6 +101,7 @@ class ACP:
|
|
|
97
101
|
mcp_urls: dict[str, str] | None = None,
|
|
98
102
|
system_prompt: str | None = None,
|
|
99
103
|
session_path: str | None = None,
|
|
104
|
+
session_meta: dict | None = None,
|
|
100
105
|
allow_empty_tool_reply: bool = False,
|
|
101
106
|
) -> ProgramResult:
|
|
102
107
|
if prompt is None:
|
|
@@ -112,6 +117,7 @@ class ACP:
|
|
|
112
117
|
"mcp_urls": mcp_urls or {},
|
|
113
118
|
"system_prompt": system_prompt or "",
|
|
114
119
|
"session_path": session_path,
|
|
120
|
+
"session_meta": session_meta or {},
|
|
115
121
|
"allow_empty_tool_reply": allow_empty_tool_reply,
|
|
116
122
|
}
|
|
117
123
|
program = await runtime.prepare_uv_script(
|
|
@@ -175,12 +181,14 @@ class ACPHarnessSession(HarnessSession):
|
|
|
175
181
|
command: list[str],
|
|
176
182
|
prompt: str | Messages | None,
|
|
177
183
|
system_prompt: str | None,
|
|
184
|
+
session_meta: dict | None,
|
|
178
185
|
) -> None:
|
|
179
186
|
super().__init__(harness, ctx, trace, runtime, endpoint, secret, mcp_urls, data)
|
|
180
187
|
self.env = env
|
|
181
188
|
self.command = command
|
|
182
189
|
self.prompt = prompt
|
|
183
190
|
self.system_prompt = system_prompt
|
|
191
|
+
self.session_meta = session_meta or {}
|
|
184
192
|
self._process: RuntimeProcess | None = None
|
|
185
193
|
self._reader: _PacketReader | None = None
|
|
186
194
|
self._stderr_tail = bytearray()
|
|
@@ -221,6 +229,7 @@ class ACPHarnessSession(HarnessSession):
|
|
|
221
229
|
"mcp_urls": self.mcp_urls,
|
|
222
230
|
"system_prompt": self.system_prompt or "",
|
|
223
231
|
"session_path": None,
|
|
232
|
+
"session_meta": self.session_meta,
|
|
224
233
|
}
|
|
225
234
|
async with self._lock:
|
|
226
235
|
if self._closed:
|
verifiers/v1/acp/runner.py
CHANGED
|
@@ -196,28 +196,38 @@ async def run_once(config: dict) -> str:
|
|
|
196
196
|
*command[1:],
|
|
197
197
|
env=os.environ.copy(),
|
|
198
198
|
transport_kwargs={"stderr": None},
|
|
199
|
-
) as
|
|
199
|
+
) as agent_process:
|
|
200
|
+
connection = agent_process[0]
|
|
200
201
|
initialized = await connection.initialize(
|
|
201
202
|
protocol_version=PROTOCOL_VERSION,
|
|
202
203
|
client_capabilities=ClientCapabilities(),
|
|
203
204
|
)
|
|
204
205
|
capabilities = initialized.agent_capabilities
|
|
205
206
|
session_path = Path(config["session_path"]) if config["session_path"] else None
|
|
207
|
+
session_meta = config["session_meta"]
|
|
206
208
|
is_new = session_path is None or not session_path.exists()
|
|
207
209
|
servers = mcp_servers(config)
|
|
208
210
|
if is_new:
|
|
209
|
-
session = await connection.new_session(
|
|
211
|
+
session = await connection.new_session(
|
|
212
|
+
cwd=os.getcwd(), mcp_servers=servers, **session_meta
|
|
213
|
+
)
|
|
210
214
|
session_id = session.session_id
|
|
211
215
|
else:
|
|
212
216
|
session_id = session_path.read_text().strip()
|
|
213
217
|
session_capabilities = capabilities and capabilities.session_capabilities
|
|
214
218
|
if session_capabilities and session_capabilities.resume is not None:
|
|
215
219
|
await connection.resume_session(
|
|
216
|
-
cwd=os.getcwd(),
|
|
220
|
+
cwd=os.getcwd(),
|
|
221
|
+
session_id=session_id,
|
|
222
|
+
mcp_servers=servers,
|
|
223
|
+
**session_meta,
|
|
217
224
|
)
|
|
218
225
|
elif capabilities and capabilities.load_session:
|
|
219
226
|
await connection.load_session(
|
|
220
|
-
cwd=os.getcwd(),
|
|
227
|
+
cwd=os.getcwd(),
|
|
228
|
+
session_id=session_id,
|
|
229
|
+
mcp_servers=servers,
|
|
230
|
+
**session_meta,
|
|
221
231
|
)
|
|
222
232
|
else:
|
|
223
233
|
raise RuntimeError("ACP agent does not support resuming sessions")
|
|
@@ -251,12 +261,13 @@ class LiveACPSession:
|
|
|
251
261
|
self.command: list[str] | None = None
|
|
252
262
|
self.server_urls: dict[str, str] | None = None
|
|
253
263
|
self.system_prompt: str | None = None
|
|
264
|
+
self.session_meta: dict | None = None
|
|
254
265
|
self.is_new = True
|
|
255
266
|
|
|
256
267
|
async def start(self, config: dict) -> None:
|
|
257
268
|
command = config["command"]
|
|
258
269
|
try:
|
|
259
|
-
|
|
270
|
+
agent_process = await self.stack.enter_async_context(
|
|
260
271
|
spawn_agent_process(
|
|
261
272
|
self.client,
|
|
262
273
|
command[0],
|
|
@@ -265,13 +276,16 @@ class LiveACPSession:
|
|
|
265
276
|
transport_kwargs={"stderr": None},
|
|
266
277
|
)
|
|
267
278
|
)
|
|
279
|
+
self.connection = agent_process[0]
|
|
268
280
|
initialized = await self.connection.initialize(
|
|
269
281
|
protocol_version=PROTOCOL_VERSION,
|
|
270
282
|
client_capabilities=ClientCapabilities(),
|
|
271
283
|
)
|
|
272
284
|
self.capabilities = initialized.agent_capabilities
|
|
273
285
|
session = await self.connection.new_session(
|
|
274
|
-
cwd=os.getcwd(),
|
|
286
|
+
cwd=os.getcwd(),
|
|
287
|
+
mcp_servers=mcp_servers(config),
|
|
288
|
+
**config["session_meta"],
|
|
275
289
|
)
|
|
276
290
|
except BaseException:
|
|
277
291
|
with suppress(BaseException):
|
|
@@ -282,6 +296,7 @@ class LiveACPSession:
|
|
|
282
296
|
self.command = command
|
|
283
297
|
self.server_urls = config["mcp_urls"]
|
|
284
298
|
self.system_prompt = config["system_prompt"]
|
|
299
|
+
self.session_meta = config["session_meta"]
|
|
285
300
|
self.is_new = True
|
|
286
301
|
|
|
287
302
|
async def run(self, config: dict) -> str:
|
|
@@ -291,6 +306,7 @@ class LiveACPSession:
|
|
|
291
306
|
config["command"] != self.command
|
|
292
307
|
or config["mcp_urls"] != self.server_urls
|
|
293
308
|
or config["system_prompt"] != self.system_prompt
|
|
309
|
+
or config["session_meta"] != self.session_meta
|
|
294
310
|
):
|
|
295
311
|
raise RuntimeError("ACP session configuration changed")
|
|
296
312
|
assert self.session_id is not None
|
|
@@ -1,56 +1,116 @@
|
|
|
1
|
-
"""Run Claude Code
|
|
1
|
+
"""Run Claude Code through the Claude Agent SDK ACP adapter."""
|
|
2
2
|
|
|
3
|
-
import json
|
|
4
3
|
import shlex
|
|
5
4
|
|
|
6
|
-
from
|
|
7
|
-
|
|
5
|
+
from verifiers.v1.acp import ACP
|
|
8
6
|
from verifiers.v1.clients import ModelContext
|
|
9
7
|
from verifiers.v1.configs.harness import HarnessConfig
|
|
10
|
-
from verifiers.v1.harness import Harness
|
|
8
|
+
from verifiers.v1.harness import Harness, HarnessSession
|
|
9
|
+
from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
|
|
11
10
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
12
11
|
from verifiers.v1.task import TaskData
|
|
13
12
|
from verifiers.v1.trace import Trace
|
|
14
13
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp"
|
|
15
|
+
PACKAGES_DIR = f"{CLAUDE_ACP_DIR}/packages"
|
|
16
|
+
ACP_VERSION = "0.63.0"
|
|
17
|
+
ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp"
|
|
18
|
+
ACP_COMMAND = [f"{NODE_BIN_DIR}/node", ACP_BIN]
|
|
19
|
+
CLAUDE_CONFIG_ROOT = ".vf-claude"
|
|
20
|
+
SKILLS_DIR = ".claude/skills"
|
|
21
|
+
ACP_INSTALL = r"""
|
|
20
22
|
set -e
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
export PATH="/var/tmp/vf-node/bin:$PATH"
|
|
24
|
+
if [ "$(cat /var/tmp/vf-claude-agent-acp/.version 2>/dev/null)" = "$VF_CLAUDE_ACP_VERSION" ] \
|
|
25
|
+
&& [ -x /var/tmp/vf-claude-agent-acp/packages/node_modules/.bin/claude-agent-acp ]; then
|
|
26
|
+
exit 0
|
|
27
|
+
fi
|
|
28
|
+
npm install --prefix /var/tmp/vf-claude-agent-acp/packages --ignore-scripts --no-audit --no-fund \
|
|
29
|
+
--omit=dev \
|
|
30
|
+
"@agentclientprotocol/claude-agent-acp@$VF_CLAUDE_ACP_VERSION" >/dev/null
|
|
31
|
+
printf %s "$VF_CLAUDE_ACP_VERSION" > /var/tmp/vf-claude-agent-acp/.version
|
|
23
32
|
"""
|
|
24
33
|
|
|
34
|
+
CLAUDE_ACP = ACP()
|
|
35
|
+
|
|
25
36
|
|
|
26
37
|
class ClaudeCodeHarnessConfig(HarnessConfig):
|
|
27
|
-
|
|
28
|
-
"""Claude Code release to install; pinned for reproducibility."""
|
|
38
|
+
pass
|
|
29
39
|
|
|
30
40
|
|
|
31
41
|
class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
|
|
32
42
|
APPENDS_SYSTEM_PROMPT = True
|
|
33
43
|
SUPPORTS_MCP = True
|
|
34
|
-
|
|
35
|
-
SUPPORTS_RESUME = False
|
|
44
|
+
SUPPORTS_RESUME = True
|
|
36
45
|
SUPPORTS_SKILLS = True
|
|
37
46
|
|
|
38
47
|
async def setup(self, runtime: Runtime) -> None:
|
|
39
48
|
await self.install_skills(runtime, SKILLS_DIR)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
+
await ensure_node(runtime)
|
|
50
|
+
acp_guarded = (
|
|
51
|
+
f"mkdir -p {CLAUDE_ACP_DIR} && "
|
|
52
|
+
f'"$(command -v flock || command -v lockf)" {CLAUDE_ACP_DIR}/install.lock '
|
|
53
|
+
f"sh -c {shlex.quote(ACP_INSTALL)}"
|
|
54
|
+
)
|
|
55
|
+
acp_result = await runtime.run(
|
|
56
|
+
["sh", "-c", acp_guarded],
|
|
57
|
+
{
|
|
58
|
+
**self.config.resolved_env,
|
|
59
|
+
"VF_CLAUDE_ACP_VERSION": ACP_VERSION,
|
|
60
|
+
},
|
|
61
|
+
)
|
|
62
|
+
if acp_result.exit_code != 0:
|
|
63
|
+
detail = (acp_result.stderr or acp_result.stdout).strip()[-500:]
|
|
64
|
+
raise RuntimeError(f"Claude Agent ACP install failed: {detail}")
|
|
65
|
+
await CLAUDE_ACP.setup(self, runtime)
|
|
66
|
+
|
|
67
|
+
async def session(
|
|
68
|
+
self,
|
|
69
|
+
ctx: ModelContext,
|
|
70
|
+
trace: Trace,
|
|
71
|
+
runtime: Runtime,
|
|
72
|
+
endpoint: str,
|
|
73
|
+
secret: str,
|
|
74
|
+
mcp_urls: dict[str, str],
|
|
75
|
+
data: TaskData,
|
|
76
|
+
) -> HarnessSession:
|
|
77
|
+
if not runtime.supports_live_processes:
|
|
78
|
+
return await super().session(
|
|
79
|
+
ctx, trace, runtime, endpoint, secret, mcp_urls, data
|
|
80
|
+
)
|
|
81
|
+
system_prompt, prompt = self.resolve_prompt(data)
|
|
82
|
+
config_dir = self.config_dir(trace)
|
|
83
|
+
options: dict[str, object] = {
|
|
84
|
+
"strictMcpConfig": True,
|
|
85
|
+
"disallowedTools": self.config.disabled_tools or [],
|
|
86
|
+
}
|
|
87
|
+
session_meta: dict[str, object] = {"claudeCode": {"options": options}}
|
|
88
|
+
if system_prompt:
|
|
89
|
+
session_meta["systemPrompt"] = {"append": system_prompt}
|
|
90
|
+
env = {
|
|
91
|
+
**self.config.resolved_env,
|
|
92
|
+
"ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
|
|
93
|
+
"ANTHROPIC_API_KEY": secret,
|
|
94
|
+
"ANTHROPIC_MODEL": ctx.model,
|
|
95
|
+
"CLAUDE_CONFIG_DIR": config_dir,
|
|
96
|
+
"CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
|
|
97
|
+
"DISABLE_AUTOUPDATER": "1",
|
|
98
|
+
"IS_SANDBOX": "1",
|
|
99
|
+
}
|
|
100
|
+
return CLAUDE_ACP.session(
|
|
101
|
+
self,
|
|
102
|
+
ctx,
|
|
103
|
+
trace,
|
|
104
|
+
runtime,
|
|
105
|
+
endpoint,
|
|
106
|
+
secret,
|
|
107
|
+
mcp_urls,
|
|
108
|
+
data,
|
|
109
|
+
env=env,
|
|
110
|
+
command=ACP_COMMAND,
|
|
111
|
+
prompt=prompt or "",
|
|
112
|
+
session_meta=session_meta,
|
|
49
113
|
)
|
|
50
|
-
result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env)
|
|
51
|
-
if result.exit_code != 0:
|
|
52
|
-
detail = (result.stderr or result.stdout).strip()[-500:]
|
|
53
|
-
raise RuntimeError(f"Claude Code install failed: {detail}")
|
|
54
114
|
|
|
55
115
|
async def launch(
|
|
56
116
|
self,
|
|
@@ -62,44 +122,44 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
|
|
|
62
122
|
mcp_urls: dict[str, str],
|
|
63
123
|
data: TaskData,
|
|
64
124
|
) -> ProgramResult:
|
|
65
|
-
system_prompt,
|
|
125
|
+
system_prompt, prompt = self.resolve_prompt(data)
|
|
126
|
+
config_dir = self.config_dir(trace)
|
|
127
|
+
|
|
128
|
+
options: dict[str, object] = {
|
|
129
|
+
"strictMcpConfig": True,
|
|
130
|
+
"disallowedTools": self.config.disabled_tools or [],
|
|
131
|
+
}
|
|
132
|
+
session_meta: dict[str, object] = {"claudeCode": {"options": options}}
|
|
133
|
+
if system_prompt:
|
|
134
|
+
session_meta["systemPrompt"] = {"append": system_prompt}
|
|
66
135
|
env = {
|
|
67
136
|
**self.config.resolved_env,
|
|
68
137
|
# Claude appends /v1/messages; give it the interception root, not the model endpoint.
|
|
69
138
|
"ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
|
|
70
139
|
"ANTHROPIC_API_KEY": secret,
|
|
71
|
-
"
|
|
140
|
+
"ANTHROPIC_MODEL": ctx.model,
|
|
141
|
+
"CLAUDE_CONFIG_DIR": config_dir,
|
|
72
142
|
"CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
|
|
73
143
|
"DISABLE_AUTOUPDATER": "1",
|
|
74
144
|
"IS_SANDBOX": "1",
|
|
75
145
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
"
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
mcp_path = f"{CLAUDE_CONFIG_DIR}/mcp.json"
|
|
97
|
-
await runtime.write(mcp_path, json.dumps(mcp).encode())
|
|
98
|
-
argv += [
|
|
99
|
-
"--mcp-config",
|
|
100
|
-
mcp_path,
|
|
101
|
-
"--strict-mcp-config",
|
|
102
|
-
"--",
|
|
103
|
-
instruction or "",
|
|
104
|
-
]
|
|
105
|
-
return await runtime.run_program(argv, env)
|
|
146
|
+
return await CLAUDE_ACP.run(
|
|
147
|
+
runtime,
|
|
148
|
+
env,
|
|
149
|
+
ACP_COMMAND,
|
|
150
|
+
prompt or "",
|
|
151
|
+
mcp_urls=mcp_urls,
|
|
152
|
+
session_path=f"{config_dir}/acp-session",
|
|
153
|
+
session_meta=session_meta,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
|
|
157
|
+
result = await runtime.run(["rm", "-rf", self.config_dir(trace)], {})
|
|
158
|
+
if result.exit_code != 0:
|
|
159
|
+
raise RuntimeError(
|
|
160
|
+
f"failed to clean up Claude config: {result.stderr.strip()[-500:]}"
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
@staticmethod
|
|
164
|
+
def config_dir(trace: Trace) -> str:
|
|
165
|
+
return f"{CLAUDE_CONFIG_ROOT}/{trace.id}"
|
|
@@ -1,9 +1,5 @@
|
|
|
1
|
-
"""Codex
|
|
1
|
+
"""Run Codex through its App Server-backed ACP adapter."""
|
|
2
2
|
|
|
3
|
-
The static musl binary runs in Linux containers without additional runtime dependencies.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
import base64
|
|
7
3
|
import hashlib
|
|
8
4
|
import json
|
|
9
5
|
import logging
|
|
@@ -11,39 +7,47 @@ import re
|
|
|
11
7
|
import shlex
|
|
12
8
|
from collections import Counter
|
|
13
9
|
|
|
10
|
+
from verifiers.v1.acp import ACP
|
|
14
11
|
from verifiers.v1.clients import ModelContext
|
|
15
12
|
from verifiers.v1.configs.harness import HarnessConfig
|
|
16
|
-
from verifiers.v1.harness import Harness
|
|
13
|
+
from verifiers.v1.harness import Harness, HarnessSession
|
|
14
|
+
from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
|
|
17
15
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
18
16
|
from verifiers.v1.task import TaskData
|
|
19
17
|
from verifiers.v1.trace import Trace
|
|
20
|
-
from verifiers.v1.types import TextContentPart
|
|
21
18
|
|
|
22
19
|
logger = logging.getLogger(__name__)
|
|
23
20
|
|
|
24
|
-
# The provider id we register on the fly via `-c` overrides — arbitrary, internal to codex.
|
|
25
21
|
PROVIDER = "intercept"
|
|
26
|
-
# The env var codex reads the provider api key (its bearer = the session secret) from.
|
|
27
22
|
KEY_VAR = "CODEX_INTERCEPT_KEY"
|
|
28
23
|
|
|
29
|
-
CODEX_DIR = "/tmp/vf-codex"
|
|
30
|
-
|
|
24
|
+
CODEX_DIR = "/var/tmp/vf-codex"
|
|
25
|
+
PACKAGES_DIR = f"{CODEX_DIR}/acp"
|
|
26
|
+
CODEX_VERSION = "0.145.0"
|
|
27
|
+
ACP_VERSION = "1.1.7"
|
|
28
|
+
ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp"
|
|
29
|
+
ACP_COMMAND = [f"{NODE_BIN_DIR}/node", ACP_BIN]
|
|
31
30
|
SKILLS_DIR = ".agents/skills"
|
|
32
31
|
INSTALL = r"""
|
|
33
32
|
set -e
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
33
|
+
export PATH="/var/tmp/vf-node/bin:$PATH"
|
|
34
|
+
versions="$VF_CODEX_VERSION:$VF_CODEX_ACP_VERSION"
|
|
35
|
+
if [ "$(cat /var/tmp/vf-codex/.versions 2>/dev/null)" = "$versions" ] \
|
|
36
|
+
&& [ -x /var/tmp/vf-codex/acp/node_modules/.bin/codex ] \
|
|
37
|
+
&& [ -x /var/tmp/vf-codex/acp/node_modules/.bin/codex-acp ]; then
|
|
38
|
+
exit 0
|
|
39
|
+
fi
|
|
40
|
+
npm install --prefix /var/tmp/vf-codex/acp --ignore-scripts --no-audit --no-fund \
|
|
41
|
+
--omit=dev \
|
|
42
|
+
"@agentclientprotocol/codex-acp@$VF_CODEX_ACP_VERSION" \
|
|
43
|
+
"@openai/codex@$VF_CODEX_VERSION" >/dev/null
|
|
44
|
+
printf %s "$versions" > /var/tmp/vf-codex/.versions
|
|
41
45
|
"""
|
|
42
46
|
|
|
47
|
+
CODEX_ACP = ACP()
|
|
48
|
+
|
|
43
49
|
|
|
44
50
|
class CodexHarnessConfig(HarnessConfig):
|
|
45
|
-
version: str = "0.144.5"
|
|
46
|
-
"""Codex release to install (the `rust-v<version>` GitHub release); pinned for reproducibility."""
|
|
47
51
|
multi_agent: bool = False
|
|
48
52
|
"""Enable Codex's native multi-agent v2 tools."""
|
|
49
53
|
|
|
@@ -56,22 +60,30 @@ class CodexHarness(Harness[CodexHarnessConfig]):
|
|
|
56
60
|
|
|
57
61
|
async def setup(self, runtime: Runtime) -> None:
|
|
58
62
|
await self.install_skills(runtime, SKILLS_DIR)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
await ensure_node(runtime)
|
|
64
|
+
logger.info(
|
|
65
|
+
"codex: ensuring Codex %s and codex-acp %s are installed",
|
|
66
|
+
CODEX_VERSION,
|
|
67
|
+
ACP_VERSION,
|
|
64
68
|
)
|
|
65
|
-
ensure = shlex.quote(f"[ -x {CODEX_BIN} ] || ({script})")
|
|
66
|
-
# Shared local runtimes may provision concurrently; only the first downloads.
|
|
67
69
|
guarded = (
|
|
68
|
-
f"mkdir -p {CODEX_DIR} &&
|
|
70
|
+
f"mkdir -p {CODEX_DIR} && "
|
|
71
|
+
f'"$(command -v flock || command -v lockf)" {CODEX_DIR}/install.lock '
|
|
72
|
+
f"sh -c {shlex.quote(INSTALL)}"
|
|
73
|
+
)
|
|
74
|
+
install = await runtime.run(
|
|
75
|
+
["sh", "-c", guarded],
|
|
76
|
+
{
|
|
77
|
+
**self.config.resolved_env,
|
|
78
|
+
"VF_CODEX_VERSION": CODEX_VERSION,
|
|
79
|
+
"VF_CODEX_ACP_VERSION": ACP_VERSION,
|
|
80
|
+
},
|
|
69
81
|
)
|
|
70
|
-
install = await runtime.run(["sh", "-c", guarded], {})
|
|
71
82
|
if install.exit_code != 0:
|
|
72
83
|
raise RuntimeError(f"codex install failed: {install.stderr.strip()[-500:]}")
|
|
84
|
+
await CODEX_ACP.setup(self, runtime)
|
|
73
85
|
|
|
74
|
-
async def
|
|
86
|
+
async def session(
|
|
75
87
|
self,
|
|
76
88
|
ctx: ModelContext,
|
|
77
89
|
trace: Trace,
|
|
@@ -80,77 +92,32 @@ class CodexHarness(Harness[CodexHarnessConfig]):
|
|
|
80
92
|
secret: str,
|
|
81
93
|
mcp_urls: dict[str, str],
|
|
82
94
|
data: TaskData,
|
|
83
|
-
) ->
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
and task.prompt is not None
|
|
88
|
-
and not isinstance(task.prompt, str)
|
|
89
|
-
):
|
|
90
|
-
system_prompt, prompt = task.system_prompt, task.prompt
|
|
91
|
-
else:
|
|
92
|
-
system_prompt, prompt = self.resolve_prompt(task)
|
|
93
|
-
image_args: list[str] = []
|
|
94
|
-
image_dir = f".vf-codex-images-{trace.id}"
|
|
95
|
-
if prompt is not None and not isinstance(prompt, str):
|
|
96
|
-
# Codex seeds one initial turn, so Messages system text joins its prompt.
|
|
97
|
-
texts = [system_prompt] if system_prompt else []
|
|
98
|
-
image_index = 0
|
|
99
|
-
for message in prompt:
|
|
100
|
-
if message.role not in ("system", "user"):
|
|
101
|
-
raise ValueError(
|
|
102
|
-
"codex exec only supports system and user initial messages"
|
|
103
|
-
)
|
|
104
|
-
parts = (
|
|
105
|
-
[TextContentPart(text=message.content)]
|
|
106
|
-
if isinstance(message.content, str)
|
|
107
|
-
else message.content
|
|
108
|
-
)
|
|
109
|
-
for part in parts:
|
|
110
|
-
if isinstance(part, TextContentPart):
|
|
111
|
-
texts.append(part.text)
|
|
112
|
-
continue
|
|
113
|
-
metadata, separator, encoded = part.image_url.url.partition(",")
|
|
114
|
-
media_type, *parameters = metadata.removeprefix("data:").split(";")
|
|
115
|
-
if (
|
|
116
|
-
not separator
|
|
117
|
-
or not metadata.startswith("data:image/")
|
|
118
|
-
or not any(p.lower() == "base64" for p in parameters)
|
|
119
|
-
):
|
|
120
|
-
raise ValueError(
|
|
121
|
-
"codex image prompts require base64 data:image URLs"
|
|
122
|
-
)
|
|
123
|
-
extension = re.sub(
|
|
124
|
-
r"[^a-zA-Z0-9]+", "_", media_type.removeprefix("image/")
|
|
125
|
-
).strip("_")
|
|
126
|
-
path = f"{image_dir}/image_{image_index}.{extension or 'image'}"
|
|
127
|
-
await runtime.write(path, base64.b64decode(encoded))
|
|
128
|
-
image_args += ["-i", path]
|
|
129
|
-
image_index += 1
|
|
130
|
-
prompt = "\n\n".join(texts)
|
|
131
|
-
argv = [
|
|
132
|
-
CODEX_BIN,
|
|
133
|
-
"exec",
|
|
134
|
-
*self._config_args(ctx, endpoint, mcp_urls),
|
|
135
|
-
*image_args,
|
|
136
|
-
"--",
|
|
137
|
-
prompt,
|
|
138
|
-
]
|
|
139
|
-
try:
|
|
140
|
-
return await runtime.run_program(
|
|
141
|
-
argv, await self._env(trace, runtime, secret)
|
|
95
|
+
) -> HarnessSession:
|
|
96
|
+
if not runtime.supports_live_processes:
|
|
97
|
+
return await super().session(
|
|
98
|
+
ctx, trace, runtime, endpoint, secret, mcp_urls, data
|
|
142
99
|
)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
100
|
+
if data.system_prompt is not None and not isinstance(data.prompt, str):
|
|
101
|
+
system_prompt, prompt = data.system_prompt, data.prompt
|
|
102
|
+
else:
|
|
103
|
+
system_prompt, prompt = self.resolve_prompt(data)
|
|
104
|
+
env = await self.build_env(ctx, trace, runtime, endpoint, secret, mcp_urls)
|
|
105
|
+
return CODEX_ACP.session(
|
|
106
|
+
self,
|
|
107
|
+
ctx,
|
|
108
|
+
trace,
|
|
109
|
+
runtime,
|
|
110
|
+
endpoint,
|
|
111
|
+
secret,
|
|
112
|
+
{},
|
|
113
|
+
data,
|
|
114
|
+
env=env,
|
|
115
|
+
command=ACP_COMMAND,
|
|
116
|
+
prompt=prompt,
|
|
117
|
+
system_prompt=system_prompt,
|
|
118
|
+
)
|
|
152
119
|
|
|
153
|
-
async def
|
|
120
|
+
async def launch(
|
|
154
121
|
self,
|
|
155
122
|
ctx: ModelContext,
|
|
156
123
|
trace: Trace,
|
|
@@ -159,120 +126,71 @@ class CodexHarness(Harness[CodexHarnessConfig]):
|
|
|
159
126
|
secret: str,
|
|
160
127
|
mcp_urls: dict[str, str],
|
|
161
128
|
data: TaskData,
|
|
162
|
-
messages,
|
|
163
129
|
) -> ProgramResult:
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
)
|
|
180
|
-
|
|
181
|
-
if not isinstance(part, TextContentPart):
|
|
182
|
-
raise TypeError(
|
|
183
|
-
"codex resume supports text user turns only (images go in "
|
|
184
|
-
"the opening prompt)"
|
|
185
|
-
)
|
|
186
|
-
texts.append(part.text)
|
|
187
|
-
if trace.num_turns == 0:
|
|
188
|
-
return await self.launch(
|
|
189
|
-
ctx,
|
|
190
|
-
trace,
|
|
191
|
-
runtime,
|
|
192
|
-
endpoint,
|
|
193
|
-
secret,
|
|
194
|
-
mcp_urls,
|
|
195
|
-
data.model_copy(update={"prompt": messages}),
|
|
196
|
-
)
|
|
197
|
-
argv = [
|
|
198
|
-
CODEX_BIN,
|
|
199
|
-
"exec",
|
|
200
|
-
"resume",
|
|
201
|
-
"--last",
|
|
202
|
-
*self._config_args(ctx, endpoint, mcp_urls),
|
|
203
|
-
"--",
|
|
204
|
-
"\n\n".join(texts),
|
|
205
|
-
]
|
|
206
|
-
return await runtime.run_program(argv, await self._env(trace, runtime, secret))
|
|
130
|
+
if (
|
|
131
|
+
data.system_prompt is not None
|
|
132
|
+
and data.prompt is not None
|
|
133
|
+
and not isinstance(data.prompt, str)
|
|
134
|
+
):
|
|
135
|
+
system_prompt, prompt = data.system_prompt, data.prompt
|
|
136
|
+
else:
|
|
137
|
+
system_prompt, prompt = self.resolve_prompt(data)
|
|
138
|
+
env = await self.build_env(ctx, trace, runtime, endpoint, secret, mcp_urls)
|
|
139
|
+
return await CODEX_ACP.run(
|
|
140
|
+
runtime,
|
|
141
|
+
env,
|
|
142
|
+
ACP_COMMAND,
|
|
143
|
+
prompt,
|
|
144
|
+
system_prompt=system_prompt,
|
|
145
|
+
session_path=f"{self.trace_home(trace)}/acp-session",
|
|
146
|
+
)
|
|
207
147
|
|
|
208
148
|
async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
|
|
209
|
-
|
|
210
|
-
result = await runtime.run(["rm", "-rf", home], {})
|
|
149
|
+
result = await runtime.run(["rm", "-rf", self.trace_home(trace)], {})
|
|
211
150
|
if result.exit_code != 0:
|
|
212
151
|
raise RuntimeError(
|
|
213
152
|
f"failed to clean up Codex home: {result.stderr.strip()[-500:]}"
|
|
214
153
|
)
|
|
215
154
|
|
|
216
155
|
@staticmethod
|
|
217
|
-
def
|
|
156
|
+
def trace_home(trace: Trace) -> str:
|
|
218
157
|
return f"/tmp/vf-codex-home-{trace.id}"
|
|
219
158
|
|
|
220
|
-
async def
|
|
221
|
-
# codex authenticates to the interception server with the session secret (its
|
|
222
|
-
# provider api key) and posts Responses calls to `{endpoint}/responses`. The
|
|
223
|
-
# per-trace CODEX_HOME scopes its recorded sessions to this rollout, so
|
|
224
|
-
# `exec resume --last` continues THIS exchange and never a neighbor's.
|
|
225
|
-
# codex refuses a home that doesn't exist, so make it before every segment.
|
|
226
|
-
home = self._home(trace)
|
|
227
|
-
await runtime.run(["mkdir", "-p", home], {})
|
|
228
|
-
return {
|
|
229
|
-
**self.config.resolved_env,
|
|
230
|
-
KEY_VAR: secret,
|
|
231
|
-
"CODEX_HOME": home,
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
def _config_args(
|
|
159
|
+
async def build_env(
|
|
235
160
|
self,
|
|
236
161
|
ctx: ModelContext,
|
|
162
|
+
trace: Trace,
|
|
163
|
+
runtime: Runtime,
|
|
237
164
|
endpoint: str,
|
|
165
|
+
secret: str,
|
|
238
166
|
mcp_urls: dict[str, str],
|
|
239
|
-
) ->
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
# Set the whole table so dots in quoted server IDs are not interpreted as
|
|
248
|
-
# `-c` path separators.
|
|
167
|
+
) -> dict[str, str]:
|
|
168
|
+
home = self.trace_home(trace)
|
|
169
|
+
created = await runtime.run(["mkdir", "-p", home], {})
|
|
170
|
+
if created.exit_code != 0:
|
|
171
|
+
raise RuntimeError(
|
|
172
|
+
f"failed to create Codex home: {created.stderr.strip()[-500:]}"
|
|
173
|
+
)
|
|
174
|
+
|
|
249
175
|
mcp_config = (
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
"
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
)
|
|
259
|
-
for name, url in mcp_urls.items()
|
|
260
|
-
)
|
|
261
|
-
+ "}",
|
|
262
|
-
]
|
|
176
|
+
"mcp_servers={"
|
|
177
|
+
+ ",".join(
|
|
178
|
+
f"{json.dumps(name, ensure_ascii=False)}="
|
|
179
|
+
f"{{url={json.dumps(url, ensure_ascii=False)},required=true,"
|
|
180
|
+
f"startup_timeout_sec=60.0,tool_timeout_sec={self.config.tool_timeout}}}"
|
|
181
|
+
for name, url in mcp_urls.items()
|
|
182
|
+
)
|
|
183
|
+
+ "}"
|
|
263
184
|
if mcp_urls
|
|
264
|
-
else
|
|
185
|
+
else ""
|
|
265
186
|
)
|
|
187
|
+
await runtime.write(f"{home}/config.toml", mcp_config.encode())
|
|
266
188
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
namespace = re.sub(r"[^a-zA-Z0-9_]", "_", name) or "_"
|
|
273
|
-
namespace_bases[name] = (
|
|
274
|
-
namespace if namespace.startswith("mcp__") else f"mcp__{namespace}"
|
|
275
|
-
)
|
|
189
|
+
namespace_bases = {
|
|
190
|
+
name: (namespace if namespace.startswith("mcp__") else f"mcp__{namespace}")
|
|
191
|
+
for name in mcp_urls
|
|
192
|
+
for namespace in (re.sub(r"[^a-zA-Z0-9_]", "_", name) or "_",)
|
|
193
|
+
}
|
|
276
194
|
namespace_counts = Counter(namespace_bases.values())
|
|
277
195
|
direct_mcp_namespaces: list[str] = []
|
|
278
196
|
for name, namespace in namespace_bases.items():
|
|
@@ -286,47 +204,43 @@ class CodexHarness(Harness[CodexHarnessConfig]):
|
|
|
286
204
|
direct_mcp_namespaces.append(namespace)
|
|
287
205
|
if len(namespace) > 49:
|
|
288
206
|
direct_mcp_namespaces.append(namespace[:49])
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
"
|
|
294
|
-
"
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
"
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
"
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
"
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
if direct_mcp_namespaces
|
|
330
|
-
else []
|
|
331
|
-
),
|
|
332
|
-
]
|
|
207
|
+
|
|
208
|
+
features: dict[str, object] = {
|
|
209
|
+
"apps": False,
|
|
210
|
+
"plugins": False,
|
|
211
|
+
"multi_agent": False,
|
|
212
|
+
"multi_agent_v2": {"enabled": self.config.multi_agent},
|
|
213
|
+
**{tool: False for tool in self.config.disabled_tools or []},
|
|
214
|
+
}
|
|
215
|
+
if direct_mcp_namespaces:
|
|
216
|
+
features["code_mode"] = {
|
|
217
|
+
"direct_only_tool_namespaces": list(
|
|
218
|
+
dict.fromkeys(direct_mcp_namespaces)
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
config = {
|
|
222
|
+
"model": ctx.model,
|
|
223
|
+
"model_provider": PROVIDER,
|
|
224
|
+
"model_providers": {
|
|
225
|
+
PROVIDER: {
|
|
226
|
+
"name": PROVIDER,
|
|
227
|
+
"base_url": endpoint,
|
|
228
|
+
"env_key": KEY_VAR,
|
|
229
|
+
"wire_api": "responses",
|
|
230
|
+
"requires_openai_auth": False,
|
|
231
|
+
}
|
|
232
|
+
},
|
|
233
|
+
"features": features,
|
|
234
|
+
}
|
|
235
|
+
return {
|
|
236
|
+
**self.config.resolved_env,
|
|
237
|
+
"CODEX_API_KEY": secret,
|
|
238
|
+
KEY_VAR: secret,
|
|
239
|
+
"APP_SERVER_LOGS": f"{home}/logs",
|
|
240
|
+
"CODEX_CONFIG": json.dumps(config),
|
|
241
|
+
"CODEX_HOME": home,
|
|
242
|
+
"DEFAULT_AUTH_REQUEST": json.dumps({"methodId": "api-key"}),
|
|
243
|
+
"INITIAL_AGENT_MODE": "agent-full-access",
|
|
244
|
+
"MODEL_PROVIDER": PROVIDER,
|
|
245
|
+
"NO_BROWSER": "1",
|
|
246
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import shlex
|
|
2
|
+
|
|
3
|
+
from verifiers.v1.runtimes import Runtime
|
|
4
|
+
|
|
5
|
+
NODE_DIR = "/var/tmp/vf-node"
|
|
6
|
+
NODE_BIN_DIR = f"{NODE_DIR}/bin"
|
|
7
|
+
NODE_VERSION = "22.19.0"
|
|
8
|
+
|
|
9
|
+
INSTALL = r"""
|
|
10
|
+
set -e
|
|
11
|
+
node=/var/tmp/vf-node
|
|
12
|
+
node_ok() { "$node/bin/node" -e 'const [a,b]=process.versions.node.split(".").map(Number); process.exit(a>22 || a===22 && b>=19 ? 0 : 1)'; }
|
|
13
|
+
|
|
14
|
+
if [ -f /etc/alpine-release ]; then
|
|
15
|
+
apk add --no-cache curl ca-certificates nodejs-current npm >/dev/null
|
|
16
|
+
if ! node -e 'const [a,b]=process.versions.node.split(".").map(Number); process.exit(a>22 || a===22 && b>=19 ? 0 : 1)'; then
|
|
17
|
+
sed -E -i 's/v[0-9]+\.[0-9]+/v3.22/g' /etc/apk/repositories
|
|
18
|
+
apk upgrade --available --no-cache >/dev/null
|
|
19
|
+
apk add --no-cache nodejs-current npm >/dev/null
|
|
20
|
+
fi
|
|
21
|
+
mkdir -p "$node/bin"
|
|
22
|
+
ln -sf "$(command -v node)" "$node/bin/node"
|
|
23
|
+
ln -sf "$(command -v npm)" "$node/bin/npm"
|
|
24
|
+
else
|
|
25
|
+
command -v curl >/dev/null 2>&1 \
|
|
26
|
+
|| { apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null; }
|
|
27
|
+
case "$(uname -s)" in Linux) node_os=linux ;; Darwin) node_os=darwin ;; *) echo "unsupported os: $(uname -s)" >&2; exit 1 ;; esac
|
|
28
|
+
if [ ! -x "$node/bin/node" ] || [ "$("$node/bin/node" --version 2>/dev/null)" != "v$VF_NODE_VERSION" ]; then
|
|
29
|
+
case "$(uname -m)" in aarch64|arm64) node_arch=arm64 ;; *) node_arch=x64 ;; esac
|
|
30
|
+
rm -rf "$node"
|
|
31
|
+
mkdir -p "$node"
|
|
32
|
+
curl -fsSL "https://nodejs.org/dist/v$VF_NODE_VERSION/node-v$VF_NODE_VERSION-${node_os}-${node_arch}.tar.gz" \
|
|
33
|
+
| tar -xz -C "$node" --strip-components=1
|
|
34
|
+
fi
|
|
35
|
+
fi
|
|
36
|
+
node_ok || { echo "ACP adapters require Node.js 22.19 or newer" >&2; exit 1; }
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def ensure_node(runtime: Runtime) -> None:
|
|
41
|
+
"""Install the shared Node runtime used by ACP adapter harnesses."""
|
|
42
|
+
lock = f"{NODE_DIR}.install.lock"
|
|
43
|
+
guarded = (
|
|
44
|
+
f'until ln -s "$$" {lock} 2>/dev/null; do '
|
|
45
|
+
f"owner=$(readlink {lock}); "
|
|
46
|
+
f'if ! kill -0 "$owner" 2>/dev/null; then '
|
|
47
|
+
f'[ "$(readlink {lock})" != "$owner" ] || rm -f {lock}; fi; '
|
|
48
|
+
f"sleep 0.1; done; "
|
|
49
|
+
f'trap \'[ "$(readlink {lock})" != "$$" ] || rm -f {lock}\' EXIT; '
|
|
50
|
+
f"sh -c {shlex.quote(INSTALL)}"
|
|
51
|
+
)
|
|
52
|
+
result = await runtime.run(["sh", "-c", guarded], {"VF_NODE_VERSION": NODE_VERSION})
|
|
53
|
+
if result.exit_code != 0:
|
|
54
|
+
raise RuntimeError(f"Node.js install failed: {result.stderr.strip()[-500:]}")
|
|
@@ -8,6 +8,7 @@ from verifiers.v1.acp import ACP
|
|
|
8
8
|
from verifiers.v1.clients import ModelContext
|
|
9
9
|
from verifiers.v1.configs.harness import HarnessConfig
|
|
10
10
|
from verifiers.v1.harness import Harness
|
|
11
|
+
from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
|
|
11
12
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
12
13
|
from verifiers.v1.task import TaskData
|
|
13
14
|
from verifiers.v1.trace import Trace
|
|
@@ -18,13 +19,12 @@ PROVIDER = "intercept"
|
|
|
18
19
|
KEY_VAR = "PI_INTERCEPT_KEY"
|
|
19
20
|
HOME_VAR = "VF_PI_ORIGINAL_HOME"
|
|
20
21
|
|
|
21
|
-
PI_DIR = "/tmp/vf-pi"
|
|
22
|
+
PI_DIR = "/var/tmp/vf-pi"
|
|
22
23
|
PACKAGES_DIR = f"{PI_DIR}/mcp"
|
|
23
24
|
PI_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi"
|
|
24
25
|
SKILLS_DIR = ".agents/skills"
|
|
25
26
|
MCP_VERSION = "2.11.0"
|
|
26
27
|
ACP_VERSION = "0.0.31"
|
|
27
|
-
NODE_VERSION = "22.19.0"
|
|
28
28
|
MCP_ADAPTER = f"{PACKAGES_DIR}/node_modules/pi-mcp-adapter/index.ts"
|
|
29
29
|
ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi-acp"
|
|
30
30
|
ACP_COMMAND = [
|
|
@@ -34,39 +34,14 @@ ACP_COMMAND = [
|
|
|
34
34
|
f'export {HOME_VAR}="$HOME"; '
|
|
35
35
|
'PI_CODING_AGENT_DIR="$PWD/$PI_CODING_AGENT_DIR"; '
|
|
36
36
|
'export PI_CODING_AGENT_DIR HOME="$PI_CODING_AGENT_DIR"; '
|
|
37
|
-
f'export PATH="{
|
|
37
|
+
f'export PATH="{NODE_BIN_DIR}:$PATH"; exec {ACP_BIN}'
|
|
38
38
|
),
|
|
39
39
|
]
|
|
40
40
|
|
|
41
41
|
INSTALL = r"""
|
|
42
42
|
set -e
|
|
43
|
-
packages=/tmp/vf-pi/mcp
|
|
44
|
-
|
|
45
|
-
node_ok() { node -e 'const [a,b]=process.versions.node.split(".").map(Number); process.exit(a>22 || a===22 && b>=19 ? 0 : 1)'; }
|
|
46
|
-
|
|
47
|
-
if [ -f /etc/alpine-release ]; then
|
|
48
|
-
apk add --no-cache curl ca-certificates nodejs-current npm >/dev/null
|
|
49
|
-
if ! node_ok; then
|
|
50
|
-
sed -E -i 's/v[0-9]+\.[0-9]+/v3.22/g' /etc/apk/repositories
|
|
51
|
-
apk upgrade --available --no-cache >/dev/null
|
|
52
|
-
apk add --no-cache nodejs-current npm >/dev/null
|
|
53
|
-
fi
|
|
54
|
-
node_bin="$(dirname "$(command -v node)")"
|
|
55
|
-
else
|
|
56
|
-
command -v curl >/dev/null 2>&1 \
|
|
57
|
-
|| { apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null; }
|
|
58
|
-
case "$(uname -s)" in Linux) node_os=linux ;; Darwin) node_os=darwin ;; *) echo "unsupported os: $(uname -s)" >&2; exit 1 ;; esac
|
|
59
|
-
if [ ! -x "$node/bin/node" ] || [ "$("$node/bin/node" --version 2>/dev/null)" != "v$VF_PI_NODE_VERSION" ]; then
|
|
60
|
-
case "$(uname -m)" in aarch64|arm64) node_arch=arm64 ;; *) node_arch=x64 ;; esac
|
|
61
|
-
rm -rf "$node"
|
|
62
|
-
mkdir -p "$node"
|
|
63
|
-
curl -fsSL "https://nodejs.org/dist/v$VF_PI_NODE_VERSION/node-v$VF_PI_NODE_VERSION-${node_os}-${node_arch}.tar.gz" \
|
|
64
|
-
| tar -xz -C "$node" --strip-components=1
|
|
65
|
-
fi
|
|
66
|
-
node_bin="$node/bin"
|
|
67
|
-
fi
|
|
68
|
-
export PATH="$node_bin:$PATH"
|
|
69
|
-
node_ok || { echo "pi-acp requires Node.js 22.19 or newer" >&2; exit 1; }
|
|
43
|
+
packages=/var/tmp/vf-pi/mcp
|
|
44
|
+
export PATH="/var/tmp/vf-node/bin:$PATH"
|
|
70
45
|
|
|
71
46
|
versions="$VF_PI_VERSION:$VF_PI_MCP_VERSION:$VF_PI_ACP_VERSION"
|
|
72
47
|
if [ "$(cat "$packages/.versions" 2>/dev/null)" != "$versions" ]; then
|
|
@@ -125,6 +100,7 @@ class PiHarness(Harness[PiHarnessConfig]):
|
|
|
125
100
|
|
|
126
101
|
async def setup(self, runtime: Runtime) -> None:
|
|
127
102
|
await self.install_skills(runtime, SKILLS_DIR)
|
|
103
|
+
await ensure_node(runtime)
|
|
128
104
|
logger.info(
|
|
129
105
|
"pi: ensuring Pi %s and pi-acp %s are installed",
|
|
130
106
|
self.config.version,
|
|
@@ -147,7 +123,6 @@ class PiHarness(Harness[PiHarnessConfig]):
|
|
|
147
123
|
"VF_PI_VERSION": self.config.version,
|
|
148
124
|
"VF_PI_MCP_VERSION": MCP_VERSION,
|
|
149
125
|
"VF_PI_ACP_VERSION": ACP_VERSION,
|
|
150
|
-
"VF_PI_NODE_VERSION": NODE_VERSION,
|
|
151
126
|
},
|
|
152
127
|
)
|
|
153
128
|
if install.exit_code != 0:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.2.2.
|
|
3
|
+
Version: 0.2.2.dev91
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -183,8 +183,8 @@ verifiers/v1/task.py,sha256=jwMKiKlMtksTd8j2dcDlFtb6LC8BRE-N5jkNXlMp2jc,9457
|
|
|
183
183
|
verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
|
|
184
184
|
verifiers/v1/trace.py,sha256=Mjc9NaqmUVKXhn6RKaTaGVxPRldCGqgHxb74cqtp5_k,20026
|
|
185
185
|
verifiers/v1/types.py,sha256=1PxamJspmoTc3OlFZAwH6o_2i_Tg9ZNvHxttIHWIxq8,8948
|
|
186
|
-
verifiers/v1/acp/__init__.py,sha256=
|
|
187
|
-
verifiers/v1/acp/runner.py,sha256=
|
|
186
|
+
verifiers/v1/acp/__init__.py,sha256=RusAr7bFDHOjQc3mgqnADfhgen9bJlyLs8wQke2fUUs,11675
|
|
187
|
+
verifiers/v1/acp/runner.py,sha256=4VWM-KWS7jD2tuU_CxO4wwnYNrii6hTDawX_4q4qoLc,14730
|
|
188
188
|
verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
189
189
|
verifiers/v1/cli/debug.py,sha256=WGCiapH2CCwyPfESca5623-_csQv93PoglR8yX5-D3g,11488
|
|
190
190
|
verifiers/v1/cli/gepa.py,sha256=hLIbS_4jBx0vvy4PXoePR5riFXcNvzQCMixZ-g2OR_U,4026
|
|
@@ -247,6 +247,7 @@ verifiers/v1/gepa/dataset.py,sha256=N2QYqUljHVuAdCbUuydEIlarvAWa6LFdGKu9qsOvjro,
|
|
|
247
247
|
verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
|
|
248
248
|
verifiers/v1/gepa/runner.py,sha256=_B_YPd_h3kelbNE2029a6n3-QpqD8HOSDmOJRtX1PR8,6036
|
|
249
249
|
verifiers/v1/harnesses/__init__.py,sha256=cLyHRiH0E63WcVW9dBqR4OEqPSfPR4wQmB6gunghV4Q,1759
|
|
250
|
+
verifiers/v1/harnesses/node.py,sha256=_-7NofTl-TNuwgEg8qxrD28shlQuyBWgiVHwCCwbdVo,2410
|
|
250
251
|
verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
|
|
251
252
|
verifiers/v1/harnesses/bash/harness.py,sha256=iT-F8fEDaV0jveCOG4ZTavRahhVvbq8ehPTbfG3Cre8,5349
|
|
252
253
|
verifiers/v1/harnesses/bash/program.py,sha256=USO3fu8CyOu_qiUisGTpsFUsu5_56KLwM-JdFZP4k78,15338
|
|
@@ -254,9 +255,9 @@ verifiers/v1/harnesses/browser_use/__init__.py,sha256=yxm2IG5TDLadxZEkX8SL26XWbc
|
|
|
254
255
|
verifiers/v1/harnesses/browser_use/harness.py,sha256=pxlpyJdnIFdlXVeMCcSgMH1uGeuU8LMEG0UF8emLXU0,5900
|
|
255
256
|
verifiers/v1/harnesses/browser_use/program.py,sha256=ygC0e7iocV6bjTyqE4b3QECvQqNsxwFZDBcHOYUUfHY,14291
|
|
256
257
|
verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umuahmOSqFjTrJ2hryTw,171
|
|
257
|
-
verifiers/v1/harnesses/claude_code/harness.py,sha256=
|
|
258
|
+
verifiers/v1/harnesses/claude_code/harness.py,sha256=5eoL5yEQJD9qR9jGf1kuvu5y1E3qE8pRsirVJioOqYQ,5759
|
|
258
259
|
verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
|
|
259
|
-
verifiers/v1/harnesses/codex/harness.py,sha256=
|
|
260
|
+
verifiers/v1/harnesses/codex/harness.py,sha256=OI_B7P6OMbd4E4tR5NtR-iyrs1Sb2OG7XQzx7VXDBNo,8312
|
|
260
261
|
verifiers/v1/harnesses/hermes_agent/__init__.py,sha256=k84Y8UTTf1KOw5LGqPb2dk-hYFagPevDJBOF7qsR_vo,176
|
|
261
262
|
verifiers/v1/harnesses/hermes_agent/harness.py,sha256=AveZi9g4pga9pYy4VAmn_KMKlJw3KhwG45JZcPxbdR4,5094
|
|
262
263
|
verifiers/v1/harnesses/hermes_agent/program.py,sha256=9qPLbUnhox2bGu5ypEEnBhw9QKtYwNvrbl7W_zeNMdk,229
|
|
@@ -271,7 +272,7 @@ verifiers/v1/harnesses/null/program.py,sha256=Lapx_g-uioKt0w9r2lcfoIkC0NPAX7wwNz
|
|
|
271
272
|
verifiers/v1/harnesses/openclaw/__init__.py,sha256=kNyfPB1sGwEf4K3mTfKN0VEyxun3SGvNpvArVKvH-x8,160
|
|
272
273
|
verifiers/v1/harnesses/openclaw/harness.py,sha256=n1o7gvsEEv0OsnU_WvyEd00Q4KizWQawSwHqdvqzLnw,14745
|
|
273
274
|
verifiers/v1/harnesses/pi/__init__.py,sha256=1mSnzOf-RdEchdeg6IHtcdobG5KIpAhuTdUIgsCIn74,117
|
|
274
|
-
verifiers/v1/harnesses/pi/harness.py,sha256=
|
|
275
|
+
verifiers/v1/harnesses/pi/harness.py,sha256=Gwl-oCMuEplQP4PsrWZarNR3crcVdCbEOJtKCYSWWtc,7387
|
|
275
276
|
verifiers/v1/harnesses/pool/__init__.py,sha256=HTYsNiWGdiNEsSHj4xhPmshJSNb7SHjQ7d4yZQ4p3HM,127
|
|
276
277
|
verifiers/v1/harnesses/pool/harness.py,sha256=6DlOA2W3s-NrxdQ9GaMmZg7mSNJGz2GC21kH2Pmoy9w,4025
|
|
277
278
|
verifiers/v1/harnesses/rlm/__init__.py,sha256=hwx51xTdEWzzYgMj-p4ETNcptoub3IPMDbEdJ4uB8Ns,122
|
|
@@ -338,8 +339,8 @@ verifiers/v1/utils/platform.py,sha256=56Ixmk1SER6q5LvzyYcA0hgzGpzhxKZL8Hiqp4c_XV
|
|
|
338
339
|
verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
|
|
339
340
|
verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
|
|
340
341
|
verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
|
|
341
|
-
verifiers-0.2.2.
|
|
342
|
-
verifiers-0.2.2.
|
|
343
|
-
verifiers-0.2.2.
|
|
344
|
-
verifiers-0.2.2.
|
|
345
|
-
verifiers-0.2.2.
|
|
342
|
+
verifiers-0.2.2.dev91.dist-info/METADATA,sha256=NDm3kLyWtrNXJbEIc2HVEWBarMuI3DrubNo2EhdZAzo,4545
|
|
343
|
+
verifiers-0.2.2.dev91.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
344
|
+
verifiers-0.2.2.dev91.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
|
|
345
|
+
verifiers-0.2.2.dev91.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
|
|
346
|
+
verifiers-0.2.2.dev91.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|