premanmcp 1.1.7 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/agent.js +315 -36
- package/bin/desktop.js +2 -2
- package/bin/detect.js +29 -0
- package/bin/eval.js +430 -22
- package/bin/eval_entry.js +762 -0
- package/bin/eval_harness.py +166 -5
- package/bin/eval_runtime.js +156 -0
- package/bin/eval_target.js +160 -113
- package/bin/eval_templates.js +684 -0
- package/bin/runner.js +42 -3
- package/bin/shared.js +2 -1
- package/bin/target.js +105 -3
- package/package.json +2 -2
|
@@ -0,0 +1,684 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The wrapper, assembled here rather than written by a model.
|
|
3
|
+
*
|
|
4
|
+
* Discovery used to ask for a complete Python file back. That measured 2/6 on
|
|
5
|
+
* determinism -- three repositories named the right entry point twice and still
|
|
6
|
+
* produced different bytes each run -- because a model writing a whole file
|
|
7
|
+
* freehand has no reason to write it the same way twice. It also limited the
|
|
8
|
+
* feature to the two languages the prompt happened to explain, since the model
|
|
9
|
+
* could only produce an invocation it had been told how to write.
|
|
10
|
+
*
|
|
11
|
+
* So the model now answers `{entry_file, entry_symbol, shape}` and the CLI owns
|
|
12
|
+
* the file. For every shape below, no model writes any code, which makes the
|
|
13
|
+
* output byte-identical across runs and makes a new language a template rather
|
|
14
|
+
* than a new paragraph of prompt.
|
|
15
|
+
*
|
|
16
|
+
* `history` as the parameter name and the `{"response", "events"}` return shape
|
|
17
|
+
* become structural guarantees here instead of prompt rules checked afterwards
|
|
18
|
+
* by a probe.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/** Shapes the CLI can build with no model-written code. */
|
|
22
|
+
export const SHAPES = new Set([
|
|
23
|
+
"py_history", // f(message, history)
|
|
24
|
+
"py_plain", // f(message)
|
|
25
|
+
"py_class", // Class().method(message)
|
|
26
|
+
"langgraph", // compiled graph.invoke({"messages": [...]})
|
|
27
|
+
"node_bridge", // a JS/TS agent over a long-lived node process
|
|
28
|
+
"cli_subprocess", // a compiled or standalone binary, one process per turn
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
/** A shape the CLI cannot build: the model supplies one function body. */
|
|
32
|
+
export const FREEFORM = "other";
|
|
33
|
+
|
|
34
|
+
const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
35
|
+
|
|
36
|
+
/** Python literal for a JS string. JSON is a subset of Python for these. */
|
|
37
|
+
function lit(value) {
|
|
38
|
+
return JSON.stringify(String(value ?? ""));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Python list-of-strings literal. */
|
|
42
|
+
function litList(values) {
|
|
43
|
+
return `[${(values || []).map((v) => lit(v)).join(", ")}]`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ── The parts every wrapper shares ──────────────────────────────────────
|
|
47
|
+
|
|
48
|
+
const HEADER = `# Generated by \`preman eval target\`. Do not edit by hand.
|
|
49
|
+
# shape: %%SHAPE%% entry: %%ENTRY_DISPLAY%%
|
|
50
|
+
#
|
|
51
|
+
# Regenerate with \`preman eval target\` rather than editing: the digest in
|
|
52
|
+
# .preman.json is what proves this file is the one that was approved.
|
|
53
|
+
`;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Loading the target module.
|
|
57
|
+
*
|
|
58
|
+
* Deliberately lazy -- importing this wrapper must not start a server or open a
|
|
59
|
+
* socket, because a health probe imports it without wanting to run anything.
|
|
60
|
+
* `preman_selfcheck` is the explicit way to ask "does this still work here".
|
|
61
|
+
*
|
|
62
|
+
* The global is assigned only after `exec_module` returns. Assigning it first
|
|
63
|
+
* leaves a non-None empty module cached when a load fails, and then every turn
|
|
64
|
+
* after the first reports `has no attribute <symbol>` instead of the real import
|
|
65
|
+
* error -- which reads as a bug in the customer's file when the customer's file
|
|
66
|
+
* is fine.
|
|
67
|
+
*/
|
|
68
|
+
const PY_LOADER = String.raw`
|
|
69
|
+
import importlib.util
|
|
70
|
+
import sys
|
|
71
|
+
from pathlib import Path
|
|
72
|
+
|
|
73
|
+
_REPO = Path(__file__).resolve().parent.parent
|
|
74
|
+
_SRC = _REPO / %%ENTRY_FILE%%
|
|
75
|
+
_SYMBOL = %%ENTRY_SYMBOL%%
|
|
76
|
+
_mod = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _agent():
|
|
80
|
+
global _mod
|
|
81
|
+
if _mod is None:
|
|
82
|
+
if str(_REPO) not in sys.path:
|
|
83
|
+
sys.path.insert(0, str(_REPO))
|
|
84
|
+
spec = importlib.util.spec_from_file_location("preman_target_agent", _SRC)
|
|
85
|
+
if spec is None or spec.loader is None:
|
|
86
|
+
raise ImportError("no import spec for %s" % _SRC)
|
|
87
|
+
mod = importlib.util.module_from_spec(spec)
|
|
88
|
+
sys.modules[spec.name] = mod
|
|
89
|
+
try:
|
|
90
|
+
spec.loader.exec_module(mod)
|
|
91
|
+
except BaseException:
|
|
92
|
+
sys.modules.pop(spec.name, None)
|
|
93
|
+
raise
|
|
94
|
+
_mod = mod
|
|
95
|
+
return _mod
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _target():
|
|
99
|
+
agent = _agent()
|
|
100
|
+
try:
|
|
101
|
+
return getattr(agent, _SYMBOL)
|
|
102
|
+
except AttributeError:
|
|
103
|
+
names = ", ".join(n for n in dir(agent) if not n.startswith("_"))
|
|
104
|
+
raise AttributeError(
|
|
105
|
+
"%s defines no %r. It defines: %s" % (_SRC, _SYMBOL, names)
|
|
106
|
+
) from None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def preman_selfcheck() -> str:
|
|
110
|
+
_target()
|
|
111
|
+
return _SYMBOL
|
|
112
|
+
`;
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The reply, normalised.
|
|
116
|
+
*
|
|
117
|
+
* Owned by the CLI for every shape: the harness needs a dict with a string
|
|
118
|
+
* `response`, and an agent that returns a bare string, a dataclass or a
|
|
119
|
+
* LangGraph state dict is the normal case rather than the exception.
|
|
120
|
+
*/
|
|
121
|
+
const PY_NORMALISE = String.raw`
|
|
122
|
+
|
|
123
|
+
def _flatten(message, history):
|
|
124
|
+
"""The conversation as one string, for an agent that takes only a message.
|
|
125
|
+
|
|
126
|
+
Shared by every shape that cannot pass prior turns through: an agent which
|
|
127
|
+
is handed only the latest message answers a multi-turn case as though it
|
|
128
|
+
were the first turn, and the run still exits 0, so the score is wrong and
|
|
129
|
+
nothing says so.
|
|
130
|
+
"""
|
|
131
|
+
if not history:
|
|
132
|
+
return message
|
|
133
|
+
lines = []
|
|
134
|
+
for item in history:
|
|
135
|
+
role = (item or {}).get("role") or "user"
|
|
136
|
+
speaker = "Assistant" if role == "assistant" else "User"
|
|
137
|
+
lines.append("%s: %s" % (speaker, (item or {}).get("content", "")))
|
|
138
|
+
lines.append("User: %s" % message)
|
|
139
|
+
return "\n\n".join(lines)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _text(value):
|
|
143
|
+
"""Best-effort reply text from whatever the agent returned."""
|
|
144
|
+
if isinstance(value, str):
|
|
145
|
+
return value
|
|
146
|
+
if isinstance(value, dict):
|
|
147
|
+
for key in ("response", "output", "text", "content", "result", "answer"):
|
|
148
|
+
found = value.get(key)
|
|
149
|
+
if isinstance(found, str) and found.strip():
|
|
150
|
+
return found
|
|
151
|
+
messages = value.get("messages")
|
|
152
|
+
if isinstance(messages, list) and messages:
|
|
153
|
+
last = messages[-1]
|
|
154
|
+
content = getattr(last, "content", None)
|
|
155
|
+
if content is None and isinstance(last, dict):
|
|
156
|
+
content = last.get("content")
|
|
157
|
+
if isinstance(content, str):
|
|
158
|
+
return content
|
|
159
|
+
return str(content)
|
|
160
|
+
for attr in ("content", "text", "raw", "output"):
|
|
161
|
+
found = getattr(value, attr, None)
|
|
162
|
+
if isinstance(found, str) and found.strip():
|
|
163
|
+
return found
|
|
164
|
+
return "" if value is None else str(value)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def chat(message: str, history: list | None = None) -> dict:
|
|
168
|
+
reply, events = _turn(message, history or [])
|
|
169
|
+
return {"response": _text(reply), "events": list(events or [])}
|
|
170
|
+
`;
|
|
171
|
+
|
|
172
|
+
// ── One `_turn` per shape ───────────────────────────────────────────────
|
|
173
|
+
|
|
174
|
+
const TURNS = {
|
|
175
|
+
py_history: String.raw`
|
|
176
|
+
|
|
177
|
+
def _turn(message, history):
|
|
178
|
+
return _target()(message, history), []
|
|
179
|
+
`,
|
|
180
|
+
|
|
181
|
+
py_plain: String.raw`
|
|
182
|
+
# The agent takes no prior turns, so the conversation is flattened into the
|
|
183
|
+
# message. Dropping it silently would make a multi-turn behaviour score as
|
|
184
|
+
# though the agent had been given context it never saw.
|
|
185
|
+
|
|
186
|
+
def _turn(message, history):
|
|
187
|
+
return _target()(_flatten(message, history)), []
|
|
188
|
+
`,
|
|
189
|
+
|
|
190
|
+
py_class: String.raw`
|
|
191
|
+
# The symbol is a class: constructed once and reused, so per-turn state the
|
|
192
|
+
# agent keeps for itself survives the conversation.
|
|
193
|
+
#
|
|
194
|
+
# Whether the method accepts prior turns is decided by reading its signature,
|
|
195
|
+
# ONCE, not by calling it and catching TypeError. Two reasons, both measured:
|
|
196
|
+
# a TypeError raised from inside the method body is indistinguishable from an
|
|
197
|
+
# arity mismatch, so the retry ran the agent twice for one turn -- real provider
|
|
198
|
+
# spend and duplicated side effects; and the old fallback called method(message)
|
|
199
|
+
# with no flattening, so a stateless class silently never saw the conversation
|
|
200
|
+
# and a multi-turn case scored as though it had.
|
|
201
|
+
import inspect
|
|
202
|
+
|
|
203
|
+
_instance = None
|
|
204
|
+
_takes_history = None
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _turn(message, history):
|
|
208
|
+
global _instance, _takes_history
|
|
209
|
+
if _instance is None:
|
|
210
|
+
_instance = _target()()
|
|
211
|
+
method = getattr(_instance, %%METHOD%%)
|
|
212
|
+
if _takes_history is None:
|
|
213
|
+
try:
|
|
214
|
+
params = list(inspect.signature(method).parameters.values())
|
|
215
|
+
except (TypeError, ValueError):
|
|
216
|
+
# A C-implemented or otherwise unintrospectable callable. Assume the
|
|
217
|
+
# narrower signature and flatten, which is wrong-but-complete rather
|
|
218
|
+
# than wrong-and-silently-missing-the-conversation.
|
|
219
|
+
_takes_history = False
|
|
220
|
+
else:
|
|
221
|
+
_takes_history = len(params) >= 2 or any(
|
|
222
|
+
p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD) for p in params
|
|
223
|
+
)
|
|
224
|
+
if _takes_history:
|
|
225
|
+
return method(message, history), []
|
|
226
|
+
return method(_flatten(message, history)), []
|
|
227
|
+
`,
|
|
228
|
+
|
|
229
|
+
langgraph: String.raw`
|
|
230
|
+
# A compiled graph: invoked with the whole conversation, because a graph holds
|
|
231
|
+
# no state between invocations and re-sending the transcript is how it sees one.
|
|
232
|
+
#
|
|
233
|
+
# Messages go in as OBJECTS, not as {"role", "content"} dicts. A graph node is
|
|
234
|
+
# normally written against LangChain messages and reads \`m.content\` and
|
|
235
|
+
# \`m.type\`, which a dict does not have -- measured: passing dicts broke a
|
|
236
|
+
# fixture whose reference wrapper passed objects. The fallback below carries
|
|
237
|
+
# both shapes at once so a node written either way works.
|
|
238
|
+
try:
|
|
239
|
+
from langchain_core.messages import AIMessage, HumanMessage
|
|
240
|
+
|
|
241
|
+
def _msg(role, content):
|
|
242
|
+
return AIMessage(content) if role == "assistant" else HumanMessage(content)
|
|
243
|
+
|
|
244
|
+
except Exception:
|
|
245
|
+
|
|
246
|
+
class _Msg(dict):
|
|
247
|
+
def __init__(self, role, content):
|
|
248
|
+
super().__init__(role=role, content=content)
|
|
249
|
+
self.content = content
|
|
250
|
+
self.type = "ai" if role == "assistant" else "human"
|
|
251
|
+
|
|
252
|
+
def _msg(role, content):
|
|
253
|
+
return _Msg(role, content)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _turn(message, history):
|
|
257
|
+
messages = []
|
|
258
|
+
for item in history:
|
|
259
|
+
role = (item or {}).get("role") or "user"
|
|
260
|
+
messages.append(_msg(role, (item or {}).get("content", "")))
|
|
261
|
+
messages.append(_msg("user", message))
|
|
262
|
+
state = _target().invoke({"messages": messages})
|
|
263
|
+
return state, []
|
|
264
|
+
`,
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// ── Shapes that are not a Python import at all ──────────────────────────
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* A JS/TS agent, over one long-lived node process.
|
|
271
|
+
*
|
|
272
|
+
* The protocol lives here rather than in the prompt. It used to be a paragraph
|
|
273
|
+
* the model retyped every run, and that is the one fixture the measured baseline
|
|
274
|
+
* failed: the reference bridge passed on the same machine while the generated
|
|
275
|
+
* one did not.
|
|
276
|
+
*
|
|
277
|
+
* stdout carries the protocol, so the child's own logging has to go to stderr or
|
|
278
|
+
* the first `console.log` in the agent corrupts a reply.
|
|
279
|
+
*/
|
|
280
|
+
const NODE_BRIDGE = String.raw`
|
|
281
|
+
import json
|
|
282
|
+
import shutil
|
|
283
|
+
import subprocess
|
|
284
|
+
import threading
|
|
285
|
+
|
|
286
|
+
_REPO = Path(__file__).resolve().parent.parent
|
|
287
|
+
_BRIDGE = Path(__file__).resolve().parent / "agent_bridge.mjs"
|
|
288
|
+
# The child's stderr goes to a file rather than being inherited, so that when the
|
|
289
|
+
# bridge dies its reason can be quoted back. Inheriting it printed a perfectly
|
|
290
|
+
# clear ERR_MODULE_NOT_FOUND somewhere the caller never looked, and the wrapper
|
|
291
|
+
# reported only "closed without answering" -- true, and no use to anybody.
|
|
292
|
+
_STDERR = Path(__file__).resolve().parent / "agent_bridge.log"
|
|
293
|
+
_WANTED = %%NODE_RUNTIME%%
|
|
294
|
+
_proc = None
|
|
295
|
+
_lock = threading.Lock()
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _why() -> str:
|
|
299
|
+
"""Why the bridge stopped, taken from the most informative line it wrote.
|
|
300
|
+
|
|
301
|
+
The FIRST Error line, not the tail: a Node crash prints its message near the
|
|
302
|
+
top and then a stack and a version footer, so quoting the end of the output
|
|
303
|
+
reliably captures the least useful part of it.
|
|
304
|
+
"""
|
|
305
|
+
try:
|
|
306
|
+
text = _STDERR.read_text(errors="replace").strip()
|
|
307
|
+
except OSError:
|
|
308
|
+
return ""
|
|
309
|
+
if not text:
|
|
310
|
+
return ""
|
|
311
|
+
for line in text.splitlines():
|
|
312
|
+
stripped = line.strip()
|
|
313
|
+
if "Error" in stripped and len(stripped) > 12:
|
|
314
|
+
return ": " + stripped[:600]
|
|
315
|
+
return ": " + text[:600]
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _runtime() -> str:
|
|
319
|
+
"""The runtime that is actually installed, preferring the one named.
|
|
320
|
+
|
|
321
|
+
"tsx" and "ts-node" are the obvious answers for a .ts entry point and are
|
|
322
|
+
frequently absent -- measured: a discovery run named "tsx", it was not on
|
|
323
|
+
PATH, and the wrapper died inside subprocess with a FileNotFoundError that
|
|
324
|
+
read as a broken wrapper rather than a missing tool. Current Node strips
|
|
325
|
+
TypeScript types by itself, so falling back to "node" is usually right and
|
|
326
|
+
is always better than a traceback.
|
|
327
|
+
"""
|
|
328
|
+
for name in (_WANTED, "node"):
|
|
329
|
+
if name and shutil.which(name):
|
|
330
|
+
return name
|
|
331
|
+
raise RuntimeError(
|
|
332
|
+
"no JavaScript runtime found: tried %r and 'node'. Install Node, or "
|
|
333
|
+
"put %r on PATH." % (_WANTED, _WANTED)
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _bridge():
|
|
338
|
+
global _proc
|
|
339
|
+
if _proc is None or _proc.poll() is not None:
|
|
340
|
+
_proc = subprocess.Popen(
|
|
341
|
+
[_runtime(), str(_BRIDGE)],
|
|
342
|
+
cwd=str(_REPO),
|
|
343
|
+
stdin=subprocess.PIPE,
|
|
344
|
+
stdout=subprocess.PIPE,
|
|
345
|
+
stderr=open(_STDERR, "w"),
|
|
346
|
+
text=True,
|
|
347
|
+
bufsize=1,
|
|
348
|
+
)
|
|
349
|
+
return _proc
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def preman_selfcheck() -> str:
|
|
353
|
+
_runtime()
|
|
354
|
+
proc = _bridge()
|
|
355
|
+
if proc.poll() is not None:
|
|
356
|
+
raise RuntimeError(
|
|
357
|
+
"the node bridge exited with %s%s" % (proc.returncode, _why())
|
|
358
|
+
)
|
|
359
|
+
return %%ENTRY_SYMBOL%%
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _turn(message, history):
|
|
363
|
+
with _lock:
|
|
364
|
+
proc = _bridge()
|
|
365
|
+
proc.stdin.write(json.dumps({"message": message, "history": history}) + "\n")
|
|
366
|
+
proc.stdin.flush()
|
|
367
|
+
line = proc.stdout.readline()
|
|
368
|
+
if not line:
|
|
369
|
+
raise RuntimeError("the node bridge closed without answering%s" % _why())
|
|
370
|
+
parsed = json.loads(line)
|
|
371
|
+
if parsed.get("error"):
|
|
372
|
+
raise RuntimeError(str(parsed["error"]))
|
|
373
|
+
return parsed.get("response", ""), parsed.get("events", [])
|
|
374
|
+
`;
|
|
375
|
+
|
|
376
|
+
/** The node side. Also CLI-owned, for the same reason. */
|
|
377
|
+
const NODE_SIDE = String.raw`// Generated by `+"`preman eval target`"+String.raw`. Do not edit by hand.
|
|
378
|
+
//
|
|
379
|
+
// One JSON object per line in, one per line out. stdout is the protocol, so
|
|
380
|
+
// anything the agent prints is redirected to stderr before it can corrupt a
|
|
381
|
+
// reply.
|
|
382
|
+
import { createInterface } from "node:readline";
|
|
383
|
+
|
|
384
|
+
const out = process.stdout.write.bind(process.stdout);
|
|
385
|
+
console.log = (...args) => process.stderr.write(args.join(" ") + "\n");
|
|
386
|
+
|
|
387
|
+
const mod = await import(%%NODE_ENTRY%%);
|
|
388
|
+
const agent = mod[%%ENTRY_SYMBOL%%] ?? mod.default;
|
|
389
|
+
if (typeof agent !== "function") {
|
|
390
|
+
out(JSON.stringify({ error: %%ENTRY_SYMBOL%% + " is not a function in " + %%NODE_ENTRY%% }) + "\n");
|
|
391
|
+
process.exit(1);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
for await (const line of createInterface({ input: process.stdin })) {
|
|
395
|
+
if (!line.trim()) continue;
|
|
396
|
+
try {
|
|
397
|
+
const { message, history } = JSON.parse(line);
|
|
398
|
+
const reply = await agent(message, history ?? []);
|
|
399
|
+
const text = typeof reply === "string" ? reply : (reply?.response ?? JSON.stringify(reply));
|
|
400
|
+
out(JSON.stringify({ response: text, events: reply?.events ?? [] }) + "\n");
|
|
401
|
+
} catch (error) {
|
|
402
|
+
out(JSON.stringify({ error: String(error?.stack || error) }) + "\n");
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
`;
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* A compiled or standalone binary: one process per turn.
|
|
409
|
+
*
|
|
410
|
+
* This is what makes a language other than Python or JavaScript reachable at
|
|
411
|
+
* all. The agent is not imported, so nothing about it has to be Python -- Rust,
|
|
412
|
+
* Go, Java, Ruby and any CLI-shaped agent use this one template.
|
|
413
|
+
*
|
|
414
|
+
* One process per turn rather than a long-lived one: a CLI agent generally
|
|
415
|
+
* starts a fresh session per invocation, so prior turns are replayed as text.
|
|
416
|
+
* That is lossy and costs tokens quadratically, but it is repeatable, and an
|
|
417
|
+
* eval that is not repeatable is not measuring anything.
|
|
418
|
+
*
|
|
419
|
+
* The prompt goes on stdin, never argv: argv needs quoting, has a length limit,
|
|
420
|
+
* and shows up in process listings.
|
|
421
|
+
*/
|
|
422
|
+
const CLI_SUBPROCESS = String.raw`
|
|
423
|
+
import os
|
|
424
|
+
import shutil
|
|
425
|
+
import subprocess
|
|
426
|
+
from pathlib import Path
|
|
427
|
+
|
|
428
|
+
_REPO = Path(__file__).resolve().parent.parent
|
|
429
|
+
_ARGV = %%COMMAND%%
|
|
430
|
+
_TIMEOUT = float(os.environ.get("PREMAN_AGENT_TIMEOUT", "300"))
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _binary() -> str:
|
|
434
|
+
"""The executable, with the repository's own build output preferred."""
|
|
435
|
+
override = os.environ.get("PREMAN_AGENT_BIN")
|
|
436
|
+
if override:
|
|
437
|
+
if not Path(override).is_file():
|
|
438
|
+
raise RuntimeError("PREMAN_AGENT_BIN=%s is not a file" % override)
|
|
439
|
+
return override
|
|
440
|
+
name = _ARGV[0]
|
|
441
|
+
candidate = _REPO / name
|
|
442
|
+
if candidate.is_file():
|
|
443
|
+
return str(candidate)
|
|
444
|
+
for folder in ("target/release", "target/debug", "bin", "build", "dist"):
|
|
445
|
+
candidate = _REPO / folder / Path(name).name
|
|
446
|
+
if candidate.is_file():
|
|
447
|
+
return str(candidate)
|
|
448
|
+
found = shutil.which(name)
|
|
449
|
+
if found:
|
|
450
|
+
return found
|
|
451
|
+
raise RuntimeError(
|
|
452
|
+
"no %r executable found. Build it, put it on PATH, or set "
|
|
453
|
+
"PREMAN_AGENT_BIN to the one to use." % name
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
#: Extensions worth comparing a build against. Deliberately not every file --
|
|
458
|
+
#: a README newer than the binary says nothing about whether it is stale.
|
|
459
|
+
_SOURCE_SUFFIXES = (
|
|
460
|
+
".rs", ".go", ".java", ".kt", ".rb", ".cs", ".ex", ".exs", ".php",
|
|
461
|
+
".swift", ".c", ".h", ".cc", ".cpp", ".hpp", ".m", ".mm", ".zig",
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
_SKIP_DIRS = {
|
|
465
|
+
".git", "target", "build", "dist", "bin", "vendor", "node_modules",
|
|
466
|
+
".preman", "__pycache__",
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _staleness(binary: str) -> str:
|
|
471
|
+
""""" if the build looks current, else a sentence naming the newer file.
|
|
472
|
+
|
|
473
|
+
A binary that answers happily while being older than the code is the worst
|
|
474
|
+
outcome available here: the probe passes, the run completes, and the score
|
|
475
|
+
describes software that no longer exists. Nothing on disk links a binary to
|
|
476
|
+
the source it was built from, so the timestamp is the only evidence there
|
|
477
|
+
is -- weak, and much better than nothing.
|
|
478
|
+
"""
|
|
479
|
+
try:
|
|
480
|
+
built = Path(binary).stat().st_mtime
|
|
481
|
+
except OSError:
|
|
482
|
+
return ""
|
|
483
|
+
newest, newest_at = None, built
|
|
484
|
+
for path in _REPO.rglob("*"):
|
|
485
|
+
if path.suffix not in _SOURCE_SUFFIXES:
|
|
486
|
+
continue
|
|
487
|
+
if any(part in _SKIP_DIRS for part in path.parts):
|
|
488
|
+
continue
|
|
489
|
+
try:
|
|
490
|
+
when = path.stat().st_mtime
|
|
491
|
+
except OSError:
|
|
492
|
+
continue
|
|
493
|
+
if when > newest_at:
|
|
494
|
+
newest, newest_at = path, when
|
|
495
|
+
# A couple of minutes of slack. A build followed immediately by a formatter,
|
|
496
|
+
# a recursive copy of the tree, or a checkout that stamps every file the same
|
|
497
|
+
# time are all normal and none of them mean the binary is out of date --
|
|
498
|
+
# warning about them would train the reader to ignore the warning.
|
|
499
|
+
if newest is None or (newest_at - built) < 120:
|
|
500
|
+
return ""
|
|
501
|
+
ago = (newest_at - built) / 3600.0
|
|
502
|
+
return (
|
|
503
|
+
"%s was modified %.1f hours after %s was built, so this binary may not "
|
|
504
|
+
"be the code you are measuring. Rebuild it, or set PREMAN_AGENT_BIN."
|
|
505
|
+
% (newest.relative_to(_REPO), ago, Path(binary).name)
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def preman_selfcheck() -> str:
|
|
510
|
+
"""The resolved path, not the name that was typed.
|
|
511
|
+
|
|
512
|
+
A caller approving "mytool exec -" cannot otherwise tell whether that found
|
|
513
|
+
the repository's own build output or an unrelated mytool on PATH, and the
|
|
514
|
+
two produce equally confident answers.
|
|
515
|
+
"""
|
|
516
|
+
binary = _binary()
|
|
517
|
+
stale = _staleness(binary)
|
|
518
|
+
return binary + (" [STALE] " + stale if stale else "")
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _prompt(message, history):
|
|
522
|
+
if not history:
|
|
523
|
+
return message
|
|
524
|
+
lines = []
|
|
525
|
+
for item in history:
|
|
526
|
+
role = (item or {}).get("role") or "user"
|
|
527
|
+
speaker = "Assistant" if role == "assistant" else "User"
|
|
528
|
+
lines.append("%s: %s" % (speaker, (item or {}).get("content", "")))
|
|
529
|
+
lines.append("User: %s" % message)
|
|
530
|
+
return "\n\n".join(lines)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _turn(message, history):
|
|
534
|
+
argv = [_binary()] + list(_ARGV[1:])
|
|
535
|
+
done = subprocess.run(
|
|
536
|
+
argv,
|
|
537
|
+
input=_prompt(message, history),
|
|
538
|
+
capture_output=True,
|
|
539
|
+
text=True,
|
|
540
|
+
cwd=str(_REPO),
|
|
541
|
+
timeout=_TIMEOUT,
|
|
542
|
+
)
|
|
543
|
+
if done.returncode != 0:
|
|
544
|
+
tail = (done.stderr or "").strip()[-600:]
|
|
545
|
+
raise RuntimeError("%s exited %s: %s" % (_ARGV[0], done.returncode, tail))
|
|
546
|
+
return done.stdout.strip(), []
|
|
547
|
+
`;
|
|
548
|
+
|
|
549
|
+
// ── Assembly ────────────────────────────────────────────────────────────
|
|
550
|
+
|
|
551
|
+
export class TemplateError extends Error {}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* The wrapper for one answer, or a TemplateError naming what was unusable.
|
|
555
|
+
*
|
|
556
|
+
* Every value the model supplied is checked before it reaches a template:
|
|
557
|
+
* `entry_file` must be a member of the tree it was shown, symbols must be
|
|
558
|
+
* identifiers, and a command must be a non-empty argv. An unchecked value here
|
|
559
|
+
* would be a model writing code by choosing a string.
|
|
560
|
+
*/
|
|
561
|
+
export function buildWrapper(found, { tree = [] } = {}) {
|
|
562
|
+
const shape = String(found?.shape || "");
|
|
563
|
+
if (!SHAPES.has(shape)) {
|
|
564
|
+
throw new TemplateError(`unknown shape ${JSON.stringify(shape)}`);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const entryFile = String(found?.entry_file || "");
|
|
568
|
+
if (shape !== "cli_subprocess") {
|
|
569
|
+
if (!entryFile) throw new TemplateError("no entry_file");
|
|
570
|
+
if (tree.length && !tree.includes(entryFile)) {
|
|
571
|
+
throw new TemplateError(`entry_file ${entryFile} is not in this repository`);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
const symbol = String(found?.entry_symbol || "");
|
|
576
|
+
const needsSymbol = shape !== "cli_subprocess";
|
|
577
|
+
if (needsSymbol && !IDENT.test(symbol)) {
|
|
578
|
+
throw new TemplateError(`entry_symbol ${JSON.stringify(symbol)} is not an identifier`);
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// Checked before anything interpolates it. Building the header first meant a
|
|
582
|
+
// `command` that was a string rather than an argv threw a raw TypeError out
|
|
583
|
+
// of `.join`, and the caller only catches TemplateError -- so a bad value
|
|
584
|
+
// surfaced as a crash instead of "PreMan could not build a wrapper".
|
|
585
|
+
let command = null;
|
|
586
|
+
if (shape === "cli_subprocess") {
|
|
587
|
+
command = Array.isArray(found?.command) ? found.command.map(String) : [];
|
|
588
|
+
if (!command.length || !command[0].trim()) {
|
|
589
|
+
throw new TemplateError("cli_subprocess needs a command, e.g. [\"mytool\", \"--json\", \"-\"]");
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
const header = HEADER
|
|
594
|
+
.replace("%%SHAPE%%", shape)
|
|
595
|
+
.replace("%%ENTRY_DISPLAY%%", command ? command.join(" ") : `${entryFile}:${symbol}`);
|
|
596
|
+
|
|
597
|
+
if (shape === "cli_subprocess") {
|
|
598
|
+
return {
|
|
599
|
+
shim: header + CLI_SUBPROCESS.replace("%%COMMAND%%", litList(command)) + PY_NORMALISE,
|
|
600
|
+
bridge: "",
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
if (shape === "node_bridge") {
|
|
605
|
+
const runtime = String(found?.node_runtime || "node");
|
|
606
|
+
if (!["node", "tsx", "ts-node"].includes(runtime)) {
|
|
607
|
+
throw new TemplateError(`node_runtime ${JSON.stringify(runtime)} is not one we can run`);
|
|
608
|
+
}
|
|
609
|
+
const nodeEntry = String(found?.node_entry || entryFile);
|
|
610
|
+
if (tree.length && !tree.includes(nodeEntry)) {
|
|
611
|
+
throw new TemplateError(`node_entry ${nodeEntry} is not in this repository`);
|
|
612
|
+
}
|
|
613
|
+
const shim = header
|
|
614
|
+
+ "from pathlib import Path\n"
|
|
615
|
+
+ NODE_BRIDGE
|
|
616
|
+
.replace("%%NODE_RUNTIME%%", lit(runtime))
|
|
617
|
+
.replace("%%ENTRY_SYMBOL%%", lit(symbol))
|
|
618
|
+
+ PY_NORMALISE;
|
|
619
|
+
const bridge = NODE_SIDE
|
|
620
|
+
.replace(/%%NODE_ENTRY%%/g, lit("../" + nodeEntry))
|
|
621
|
+
.replace(/%%ENTRY_SYMBOL%%/g, lit(symbol));
|
|
622
|
+
return { shim, bridge };
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
let turn = TURNS[shape];
|
|
626
|
+
if (shape === "py_class") {
|
|
627
|
+
const method = String(found?.entry_method || "run");
|
|
628
|
+
if (!IDENT.test(method)) {
|
|
629
|
+
throw new TemplateError(`entry_method ${JSON.stringify(method)} is not an identifier`);
|
|
630
|
+
}
|
|
631
|
+
turn = turn.replace("%%METHOD%%", lit(method));
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
const shim = header
|
|
635
|
+
+ PY_LOADER
|
|
636
|
+
.replace("%%ENTRY_FILE%%", lit(entryFile))
|
|
637
|
+
.replace("%%ENTRY_SYMBOL%%", lit(symbol))
|
|
638
|
+
+ turn
|
|
639
|
+
+ PY_NORMALISE;
|
|
640
|
+
return { shim, bridge: "" };
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
/**
|
|
644
|
+
* The `other` shape: the CLI still owns everything except one function body.
|
|
645
|
+
*
|
|
646
|
+
* Even here the model writes far less than it used to -- no imports, no module
|
|
647
|
+
* cache, no reply normalisation, no `chat` signature. Only the conversion from
|
|
648
|
+
* this agent's calling convention to `(reply, events)`, which is the part that
|
|
649
|
+
* genuinely varies and the part no template can know.
|
|
650
|
+
*/
|
|
651
|
+
export function buildFreeform(found, { tree = [], turnSource = "" } = {}) {
|
|
652
|
+
const entryFile = String(found?.entry_file || "");
|
|
653
|
+
if (!entryFile) throw new TemplateError("no entry_file");
|
|
654
|
+
if (tree.length && !tree.includes(entryFile)) {
|
|
655
|
+
throw new TemplateError(`entry_file ${entryFile} is not in this repository`);
|
|
656
|
+
}
|
|
657
|
+
const symbol = String(found?.entry_symbol || "");
|
|
658
|
+
if (!IDENT.test(symbol)) {
|
|
659
|
+
throw new TemplateError(`entry_symbol ${JSON.stringify(symbol)} is not an identifier`);
|
|
660
|
+
}
|
|
661
|
+
const body = String(turnSource || "").trimEnd();
|
|
662
|
+
if (!body.trim()) throw new TemplateError("no conversion body was returned");
|
|
663
|
+
|
|
664
|
+
const header = HEADER
|
|
665
|
+
.replace("%%SHAPE%%", "other")
|
|
666
|
+
.replace("%%ENTRY_DISPLAY%%", `${entryFile}:${symbol}`);
|
|
667
|
+
// The symbol is bound by its own name before the model's body runs.
|
|
668
|
+
//
|
|
669
|
+
// Measured: a real answer for a CrewAI repository wrote
|
|
670
|
+
// `Testcrew().crew().kickoff(...)`, which is the natural thing to write and
|
|
671
|
+
// failed with `NameError: name 'Testcrew' is not defined` -- the loader
|
|
672
|
+
// exposes `_target()`, and nothing told the model that. Binding the name here
|
|
673
|
+
// means the obvious body works, and the prompt now says so as well; either
|
|
674
|
+
// alone would leave the other as a trap.
|
|
675
|
+
const shim = header
|
|
676
|
+
+ PY_LOADER
|
|
677
|
+
.replace("%%ENTRY_FILE%%", lit(entryFile))
|
|
678
|
+
.replace("%%ENTRY_SYMBOL%%", lit(symbol))
|
|
679
|
+
+ "\n\ndef _turn(message, history):\n"
|
|
680
|
+
+ ` ${symbol} = _target()\n`
|
|
681
|
+
+ body + "\n"
|
|
682
|
+
+ PY_NORMALISE;
|
|
683
|
+
return { shim, bridge: "" };
|
|
684
|
+
}
|