premanmcp 1.1.11 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/account.js +48 -0
- package/bin/cli.js +73 -14
- package/bin/connect/configs.js +20 -6
- package/bin/desktop.js +557 -79
- package/bin/eval.js +328 -50
- package/bin/eval_entry.js +9 -2
- package/bin/eval_generation_contract.py +72 -0
- package/bin/eval_generation_debug.py +279 -0
- package/bin/eval_generation_status.py +284 -0
- package/bin/eval_harness.py +198 -4
- package/bin/eval_start.js +203 -0
- package/bin/eval_target.js +20 -1
- package/bin/eval_templates.js +447 -21
- package/bin/hook.js +21 -1
- package/bin/integrations.js +20 -4
- package/bin/probe.js +323 -0
- package/bin/runner.js +23 -0
- package/bin/shared.js +331 -14
- package/package.json +2 -2
package/bin/eval_templates.js
CHANGED
|
@@ -32,6 +32,7 @@ export const SHAPES = new Set([
|
|
|
32
32
|
export const FREEFORM = "other";
|
|
33
33
|
|
|
34
34
|
const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
35
|
+
const TOOL_SYMBOL = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$/;
|
|
35
36
|
|
|
36
37
|
/** Python literal for a JS string. JSON is a subset of Python for these. */
|
|
37
38
|
function lit(value) {
|
|
@@ -43,6 +44,37 @@ function litList(values) {
|
|
|
43
44
|
return `[${(values || []).map((v) => lit(v)).join(", ")}]`;
|
|
44
45
|
}
|
|
45
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Python literal for the named tools. Every value came from a model, so each is
|
|
49
|
+
* checked here; the backend filtered the same rules, and this is the second
|
|
50
|
+
* gate the templates promise ("every value the model supplied is checked").
|
|
51
|
+
* Membership in the repository is not checked here -- the caller passes only
|
|
52
|
+
* the entry file as `tree` -- and a file that is never loaded stays "pending",
|
|
53
|
+
* which every turn reports as not visible, naming it.
|
|
54
|
+
*/
|
|
55
|
+
function toolsLiteral(tools) {
|
|
56
|
+
const list = Array.isArray(tools) ? tools : [];
|
|
57
|
+
const pairs = list.map((tool) => {
|
|
58
|
+
const file = String(tool?.file || "");
|
|
59
|
+
const symbol = String(tool?.symbol || "");
|
|
60
|
+
// The same rules as the backend's `_tools` (flowtest/services/agent_entry_search.py),
|
|
61
|
+
// plus no control characters. `lit()` quotes everything else safely.
|
|
62
|
+
if (
|
|
63
|
+
!file.endsWith(".py")
|
|
64
|
+
|| file.startsWith("/")
|
|
65
|
+
|| file.split("/").includes("..")
|
|
66
|
+
|| [...file].some((ch) => ch.charCodeAt(0) < 0x20)
|
|
67
|
+
) {
|
|
68
|
+
throw new TemplateError(`tool file ${JSON.stringify(file)} is not a repository .py path`);
|
|
69
|
+
}
|
|
70
|
+
if (!TOOL_SYMBOL.test(symbol)) {
|
|
71
|
+
throw new TemplateError(`tool symbol ${JSON.stringify(symbol)} is not a name or Class.method`);
|
|
72
|
+
}
|
|
73
|
+
return `(${lit(file)}, ${lit(symbol)})`;
|
|
74
|
+
});
|
|
75
|
+
return `[${pairs.join(", ")}]`;
|
|
76
|
+
}
|
|
77
|
+
|
|
46
78
|
// ── The parts every wrapper shares ──────────────────────────────────────
|
|
47
79
|
|
|
48
80
|
const HEADER = `# Generated by \`preman eval target\`. Do not edit by hand.
|
|
@@ -111,6 +143,312 @@ def preman_selfcheck() -> str:
|
|
|
111
143
|
return _SYMBOL
|
|
112
144
|
`;
|
|
113
145
|
|
|
146
|
+
/**
|
|
147
|
+
* Tool capture for the Python shapes (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §4).
|
|
148
|
+
*
|
|
149
|
+
* Each named tool function is replaced, once, by a recording version -- in its
|
|
150
|
+
* own module, in every loaded module holding the same object (`from x import
|
|
151
|
+
* f`), in module-level dicts and lists holding it, and on the class for a
|
|
152
|
+
* method. Replacing once rather than per turn is what keeps concurrent cases
|
|
153
|
+
* from racing each other's swaps.
|
|
154
|
+
*
|
|
155
|
+
* A turn's calls go into a context variable set in `chat()`. assert-ai runs a
|
|
156
|
+
* synchronous `chat` through `asyncio.to_thread`, which copies the context, so
|
|
157
|
+
* concurrent cases stay separate without a lock. A call made in a thread the
|
|
158
|
+
* agent starts itself sees no turn; it is not guessed onto one -- every turn
|
|
159
|
+
* open at that moment is reported not visible instead.
|
|
160
|
+
*/
|
|
161
|
+
const PY_CAPTURE = String.raw`
|
|
162
|
+
import contextvars
|
|
163
|
+
import functools
|
|
164
|
+
import inspect
|
|
165
|
+
import json
|
|
166
|
+
import threading
|
|
167
|
+
import time
|
|
168
|
+
|
|
169
|
+
_TOOLS = %%TOOLS%%
|
|
170
|
+
_turn_calls = contextvars.ContextVar("preman_turn_calls", default=None)
|
|
171
|
+
_open_turns = {}
|
|
172
|
+
_open_lock = threading.Lock()
|
|
173
|
+
_stray_calls = []
|
|
174
|
+
_installed = []
|
|
175
|
+
_missing = {}
|
|
176
|
+
_pending = {}
|
|
177
|
+
_unsupported = {}
|
|
178
|
+
_install_lock = threading.Lock()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _tool_module(file):
|
|
182
|
+
# Only a module that is already loaded, found by the file it came from.
|
|
183
|
+
# Importing it here under a name guessed from its path would run the
|
|
184
|
+
# customer's module-level code a second time, as a second copy the agent
|
|
185
|
+
# never calls -- the copy that got wrapped would then see no calls at all.
|
|
186
|
+
# A file that is not loaded yet is left pending and retried next turn.
|
|
187
|
+
if (_REPO / file).resolve() == _SRC.resolve():
|
|
188
|
+
return _agent()
|
|
189
|
+
wanted = (_REPO / file).resolve()
|
|
190
|
+
for mod in list(sys.modules.values()):
|
|
191
|
+
where = getattr(mod, "__file__", None)
|
|
192
|
+
if where:
|
|
193
|
+
try:
|
|
194
|
+
if Path(where).resolve() == wanted:
|
|
195
|
+
return mod
|
|
196
|
+
except (OSError, ValueError):
|
|
197
|
+
continue
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _jsonable(value):
|
|
202
|
+
return json.loads(json.dumps(value, ensure_ascii=False, default=str))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _record(name, original, args, kwargs, result):
|
|
206
|
+
"""Build and file the event. Must never raise into the caller ('recorded()',
|
|
207
|
+
which is standing in for the agent's own call) -- capture failing is not
|
|
208
|
+
allowed to change what the agent sees. 'default=str' covers values json
|
|
209
|
+
does not know how to serialise, but not dict keys (a tuple key) or a
|
|
210
|
+
container that contains itself, both of which raise out of 'json.dumps'
|
|
211
|
+
regardless. Each of those falls back to 'repr()'; if even that raises, the
|
|
212
|
+
call is simply not recorded rather than recorded wrong or crashing the turn.
|
|
213
|
+
"""
|
|
214
|
+
try:
|
|
215
|
+
try:
|
|
216
|
+
bound = inspect.signature(original).bind(*args, **kwargs).arguments
|
|
217
|
+
tool_args = {k: v for k, v in bound.items() if k not in ("self", "cls")}
|
|
218
|
+
except Exception:
|
|
219
|
+
tool_args = {"args": list(args), "kwargs": dict(kwargs)}
|
|
220
|
+
if isinstance(result, str):
|
|
221
|
+
content = result
|
|
222
|
+
elif inspect.isgenerator(result):
|
|
223
|
+
content = "<generator>"
|
|
224
|
+
else:
|
|
225
|
+
try:
|
|
226
|
+
content = json.dumps(result, ensure_ascii=False, default=str)
|
|
227
|
+
except Exception:
|
|
228
|
+
content = repr(result)
|
|
229
|
+
try:
|
|
230
|
+
args_out = _jsonable(tool_args)
|
|
231
|
+
except Exception:
|
|
232
|
+
args_out = {str(k): repr(v) for k, v in tool_args.items()}
|
|
233
|
+
event = {
|
|
234
|
+
"role": "tool_result",
|
|
235
|
+
"tool_name": name,
|
|
236
|
+
"tool_args": args_out,
|
|
237
|
+
"content": content,
|
|
238
|
+
}
|
|
239
|
+
except Exception:
|
|
240
|
+
with _open_lock:
|
|
241
|
+
_stray_calls.append(time.monotonic())
|
|
242
|
+
return
|
|
243
|
+
|
|
244
|
+
calls = _turn_calls.get()
|
|
245
|
+
if calls is None:
|
|
246
|
+
with _open_lock:
|
|
247
|
+
_stray_calls.append(time.monotonic())
|
|
248
|
+
else:
|
|
249
|
+
calls.append(event)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _error_text(exc):
|
|
253
|
+
# str(exc) runs the agent's own __str__, which may itself raise; a new
|
|
254
|
+
# error here would replace the tool's exception before it is re-raised.
|
|
255
|
+
name = type(exc).__name__
|
|
256
|
+
try:
|
|
257
|
+
return "error: %s: %s" % (name, exc)
|
|
258
|
+
except Exception:
|
|
259
|
+
pass
|
|
260
|
+
try:
|
|
261
|
+
return "error: %s: %r" % (name, exc)
|
|
262
|
+
except Exception:
|
|
263
|
+
return "error: %s" % name
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _recording(name, original):
|
|
267
|
+
if inspect.iscoroutinefunction(original):
|
|
268
|
+
@functools.wraps(original)
|
|
269
|
+
async def recorded(*args, **kwargs):
|
|
270
|
+
try:
|
|
271
|
+
result = await original(*args, **kwargs)
|
|
272
|
+
except BaseException as exc:
|
|
273
|
+
_record(name, original, args, kwargs, _error_text(exc))
|
|
274
|
+
raise
|
|
275
|
+
_record(name, original, args, kwargs, result)
|
|
276
|
+
return result
|
|
277
|
+
else:
|
|
278
|
+
@functools.wraps(original)
|
|
279
|
+
def recorded(*args, **kwargs):
|
|
280
|
+
try:
|
|
281
|
+
result = original(*args, **kwargs)
|
|
282
|
+
except BaseException as exc:
|
|
283
|
+
_record(name, original, args, kwargs, _error_text(exc))
|
|
284
|
+
raise
|
|
285
|
+
_record(name, original, args, kwargs, result)
|
|
286
|
+
return result
|
|
287
|
+
recorded.__preman_original__ = original
|
|
288
|
+
return recorded
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _replace_everywhere(original, recorded):
|
|
292
|
+
for mod in list(sys.modules.values()):
|
|
293
|
+
try:
|
|
294
|
+
space = getattr(mod, "__dict__", None)
|
|
295
|
+
if not isinstance(space, dict):
|
|
296
|
+
continue
|
|
297
|
+
for key, value in list(space.items()):
|
|
298
|
+
if value is original:
|
|
299
|
+
space[key] = recorded
|
|
300
|
+
elif isinstance(value, dict):
|
|
301
|
+
for inner, item in list(value.items()):
|
|
302
|
+
if item is original:
|
|
303
|
+
value[inner] = recorded
|
|
304
|
+
elif isinstance(value, list):
|
|
305
|
+
for index, item in enumerate(value):
|
|
306
|
+
if item is original:
|
|
307
|
+
value[index] = recorded
|
|
308
|
+
except Exception:
|
|
309
|
+
# One odd entry in sys.modules (a proxy whose __dict__ or .items()
|
|
310
|
+
# raises) must not fail the whole install and take every other
|
|
311
|
+
# tool down with it.
|
|
312
|
+
continue
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _install():
|
|
316
|
+
# Serialized, and the agent loaded first, unconditionally, inside the
|
|
317
|
+
# lock -- PY_LOADER's own _agent() has no lock of its own, so two threads
|
|
318
|
+
# reaching it at once can each exec_module a distinct module object, and
|
|
319
|
+
# whichever one _turn() ends up calling is not reliably the one whose
|
|
320
|
+
# functions got wrapped below. Taking the lock here and loading the agent
|
|
321
|
+
# before touching _TOOLS means every chat() (every chat() calls _install()
|
|
322
|
+
# first) either does that one load-and-wrap or blocks until it is done, so
|
|
323
|
+
# _turn() always sees the same, already-wrapped module -- concurrent first
|
|
324
|
+
# calls included.
|
|
325
|
+
with _install_lock:
|
|
326
|
+
_agent()
|
|
327
|
+
for file, symbol in _TOOLS:
|
|
328
|
+
key = "%s:%s" % (file, symbol)
|
|
329
|
+
if key in _installed:
|
|
330
|
+
continue
|
|
331
|
+
mod = _tool_module(file)
|
|
332
|
+
if mod is None:
|
|
333
|
+
_pending[key] = "module not loaded yet"
|
|
334
|
+
continue
|
|
335
|
+
_pending.pop(key, None)
|
|
336
|
+
owner_name, _, method = symbol.partition(".")
|
|
337
|
+
owner = getattr(mod, owner_name, None)
|
|
338
|
+
if owner is None:
|
|
339
|
+
_missing[key] = "%s not defined" % owner_name
|
|
340
|
+
continue
|
|
341
|
+
if method:
|
|
342
|
+
if not inspect.isclass(owner):
|
|
343
|
+
# owner_name names an instance ('support = Support()'),
|
|
344
|
+
# not the class -- setattr on it would shadow the method
|
|
345
|
+
# with a plain function on one object, and calling it would
|
|
346
|
+
# raise a missing-argument TypeError the agent never had.
|
|
347
|
+
_missing[key] = "%s.%s is not a method on a class" % (owner_name, method)
|
|
348
|
+
continue
|
|
349
|
+
raw = inspect.getattr_static(owner, method, None)
|
|
350
|
+
if raw is None:
|
|
351
|
+
_missing[key] = "%s has no %s" % (owner_name, method)
|
|
352
|
+
continue
|
|
353
|
+
if isinstance(raw, (staticmethod, classmethod)):
|
|
354
|
+
wrapped = type(raw)(_recording(method, raw.__func__))
|
|
355
|
+
elif inspect.isfunction(raw):
|
|
356
|
+
wrapped = _recording(method, raw)
|
|
357
|
+
else:
|
|
358
|
+
# A property or other descriptor: replacing it with a
|
|
359
|
+
# plain function would change what the attribute even is.
|
|
360
|
+
_missing[key] = "%s.%s is not a method on a class" % (owner_name, method)
|
|
361
|
+
continue
|
|
362
|
+
setattr(owner, method, wrapped)
|
|
363
|
+
elif not inspect.isfunction(owner):
|
|
364
|
+
# A framework tool object (LangChain/crewAI '@tool', OpenAI
|
|
365
|
+
# Agents '@function_tool'), a callable instance or a class:
|
|
366
|
+
# swapping it for a plain function would remove the attributes
|
|
367
|
+
# and type the agent relies on. Left exactly as it is.
|
|
368
|
+
_missing.pop(key, None)
|
|
369
|
+
_unsupported[key] = "not a plain function (%s)" % type(owner).__name__
|
|
370
|
+
continue
|
|
371
|
+
elif getattr(owner, "__preman_original__", None) is None:
|
|
372
|
+
_replace_everywhere(owner, _recording(owner_name, owner))
|
|
373
|
+
_missing.pop(key, None)
|
|
374
|
+
_unsupported.pop(key, None)
|
|
375
|
+
_installed.append(key)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _tool_names(installed):
|
|
379
|
+
return [key.split(":", 1)[1].split(".")[-1] for key in installed]
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _begin_turn():
|
|
383
|
+
calls = []
|
|
384
|
+
token = _turn_calls.set(calls)
|
|
385
|
+
marker = object()
|
|
386
|
+
with _open_lock:
|
|
387
|
+
_open_turns[id(marker)] = time.monotonic()
|
|
388
|
+
return token, marker, calls
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _end_turn(token, marker, calls):
|
|
392
|
+
_turn_calls.reset(token)
|
|
393
|
+
now = time.monotonic()
|
|
394
|
+
with _open_lock:
|
|
395
|
+
started = _open_turns.pop(id(marker))
|
|
396
|
+
stray = any(started <= t <= now for t in _stray_calls)
|
|
397
|
+
oldest = min(_open_turns.values(), default=now)
|
|
398
|
+
_stray_calls[:] = [t for t in _stray_calls if t >= oldest]
|
|
399
|
+
# A snapshot under the same lock _install() mutates them under, so a turn
|
|
400
|
+
# that lands between two of _install()'s steps on another thread cannot
|
|
401
|
+
# read a torn "some tools installed, matching missing entry not popped yet"
|
|
402
|
+
# state.
|
|
403
|
+
with _install_lock:
|
|
404
|
+
installed = list(_installed)
|
|
405
|
+
missing = sorted(_missing)
|
|
406
|
+
unseen = dict(_pending)
|
|
407
|
+
unseen.update(_unsupported)
|
|
408
|
+
not_watched = dict(_missing)
|
|
409
|
+
not_watched.update(unseen)
|
|
410
|
+
# "recorded" only when every named tool is installed: a tool that is not
|
|
411
|
+
# watched could have run without being seen, so a zero would not be a
|
|
412
|
+
# confirmed zero.
|
|
413
|
+
if not _TOOLS:
|
|
414
|
+
state, reason = "not_visible", "discovery named no tool functions"
|
|
415
|
+
elif not installed and not unseen:
|
|
416
|
+
state, reason = "not_visible", "no named tool function could be found"
|
|
417
|
+
elif not_watched:
|
|
418
|
+
state, reason = "not_visible", "not watched: " + "; ".join(
|
|
419
|
+
"%s (%s)" % (k, v) for k, v in sorted(not_watched.items())
|
|
420
|
+
)
|
|
421
|
+
elif stray:
|
|
422
|
+
state, reason = "not_visible", "a tool ran outside any turn, in a thread the agent started"
|
|
423
|
+
else:
|
|
424
|
+
state, reason = "recorded", ""
|
|
425
|
+
return {
|
|
426
|
+
"state": state,
|
|
427
|
+
"source": "wrapped",
|
|
428
|
+
"watched": _tool_names(installed),
|
|
429
|
+
"missing": missing,
|
|
430
|
+
"reason": reason,
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def preman_selfcheck() -> str:
|
|
435
|
+
# _install() first: it takes _install_lock and loads the agent itself
|
|
436
|
+
# before touching _TOOLS. Calling _target() first would load the agent
|
|
437
|
+
# outside that lock, racing a concurrent call's locked load.
|
|
438
|
+
_install()
|
|
439
|
+
_target()
|
|
440
|
+
# Only a missing tool fails: a pending one (its module is not loaded yet)
|
|
441
|
+
# or an unsupported one (not a plain function) is reported per turn as
|
|
442
|
+
# "not visible" instead, and the agent still runs.
|
|
443
|
+
if _missing:
|
|
444
|
+
raise RuntimeError(
|
|
445
|
+
"tool functions named by discovery were not found: "
|
|
446
|
+
+ "; ".join("%s (%s)" % (k, v) for k, v in sorted(_missing.items()))
|
|
447
|
+
+ ". Re-run 'preman eval target'."
|
|
448
|
+
)
|
|
449
|
+
return _SYMBOL
|
|
450
|
+
`;
|
|
451
|
+
|
|
114
452
|
/**
|
|
115
453
|
* The reply, normalised.
|
|
116
454
|
*
|
|
@@ -162,13 +500,41 @@ def _text(value):
|
|
|
162
500
|
if isinstance(found, str) and found.strip():
|
|
163
501
|
return found
|
|
164
502
|
return "" if value is None else str(value)
|
|
503
|
+
`;
|
|
165
504
|
|
|
505
|
+
/** `chat()` for the shapes that do not capture tool calls. */
|
|
506
|
+
const PY_CHAT = String.raw`
|
|
166
507
|
|
|
167
508
|
def chat(message: str, history: list | None = None) -> dict:
|
|
168
509
|
reply, events = _turn(message, history or [])
|
|
169
510
|
return {"response": _text(reply), "events": list(events or [])}
|
|
170
511
|
`;
|
|
171
512
|
|
|
513
|
+
/** `chat()` for `cli_subprocess`: the agent reports its own calls through a file. */
|
|
514
|
+
const PY_CHAT_REPORTED = String.raw`
|
|
515
|
+
|
|
516
|
+
def chat(message: str, history: list | None = None) -> dict:
|
|
517
|
+
reply, events, capture = _turn(message, history or [])
|
|
518
|
+
return {"response": _text(reply), "events": list(events or []), "tool_capture": capture}
|
|
519
|
+
`;
|
|
520
|
+
|
|
521
|
+
/** `chat()` for the Python shapes: the turn runs inside a capture window. */
|
|
522
|
+
const PY_CHAT_CAPTURING = String.raw`
|
|
523
|
+
|
|
524
|
+
def chat(message: str, history: list | None = None) -> dict:
|
|
525
|
+
_install()
|
|
526
|
+
token, marker, calls = _begin_turn()
|
|
527
|
+
try:
|
|
528
|
+
reply, events = _turn(message, history or [])
|
|
529
|
+
finally:
|
|
530
|
+
capture = _end_turn(token, marker, calls)
|
|
531
|
+
return {
|
|
532
|
+
"response": _text(reply),
|
|
533
|
+
"events": list(events or []) + calls,
|
|
534
|
+
"tool_capture": capture,
|
|
535
|
+
}
|
|
536
|
+
`;
|
|
537
|
+
|
|
172
538
|
// ── One `_turn` per shape ───────────────────────────────────────────────
|
|
173
539
|
|
|
174
540
|
const TURNS = {
|
|
@@ -420,9 +786,11 @@ for await (const line of createInterface({ input: process.stdin })) {
|
|
|
420
786
|
* and shows up in process listings.
|
|
421
787
|
*/
|
|
422
788
|
const CLI_SUBPROCESS = String.raw`
|
|
789
|
+
import json
|
|
423
790
|
import os
|
|
424
791
|
import shutil
|
|
425
792
|
import subprocess
|
|
793
|
+
import tempfile
|
|
426
794
|
from pathlib import Path
|
|
427
795
|
|
|
428
796
|
_REPO = Path(__file__).resolve().parent.parent
|
|
@@ -530,20 +898,64 @@ def _prompt(message, history):
|
|
|
530
898
|
return "\n\n".join(lines)
|
|
531
899
|
|
|
532
900
|
|
|
901
|
+
def _not_visible(reason):
|
|
902
|
+
return {"state": "not_visible", "source": "events_file", "watched": [], "missing": [], "reason": reason}
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
def _read_events(path):
|
|
906
|
+
"""The tool calls the agent wrote to PREMAN_TOOL_EVENTS_FILE, one JSON line each.
|
|
907
|
+
|
|
908
|
+
The agent creates the file even when it calls nothing, so a missing file
|
|
909
|
+
means "this agent does not report", never "it called no tools".
|
|
910
|
+
"""
|
|
911
|
+
if not os.path.exists(path):
|
|
912
|
+
return [], _not_visible("the agent did not create PREMAN_TOOL_EVENTS_FILE, so its tool calls are not reported")
|
|
913
|
+
events = []
|
|
914
|
+
with open(path, encoding="utf-8") as handle:
|
|
915
|
+
for line in handle:
|
|
916
|
+
line = line.strip()
|
|
917
|
+
if not line:
|
|
918
|
+
continue
|
|
919
|
+
try:
|
|
920
|
+
item = json.loads(line)
|
|
921
|
+
except ValueError:
|
|
922
|
+
return events, _not_visible("PREMAN_TOOL_EVENTS_FILE held a line that is not JSON")
|
|
923
|
+
if not isinstance(item, dict) or not item.get("tool_name"):
|
|
924
|
+
return events, _not_visible("a line in PREMAN_TOOL_EVENTS_FILE has no tool_name")
|
|
925
|
+
args = item.get("tool_args")
|
|
926
|
+
content = item.get("content", "")
|
|
927
|
+
events.append({
|
|
928
|
+
"role": "tool_result",
|
|
929
|
+
"tool_name": str(item["tool_name"]),
|
|
930
|
+
"tool_args": args if isinstance(args, dict) else {},
|
|
931
|
+
"content": content if isinstance(content, str) else json.dumps(content, ensure_ascii=False, default=str),
|
|
932
|
+
})
|
|
933
|
+
return events, {"state": "recorded", "source": "events_file", "watched": [], "missing": [], "reason": ""}
|
|
934
|
+
|
|
935
|
+
|
|
533
936
|
def _turn(message, history):
|
|
534
937
|
argv = [_binary()] + list(_ARGV[1:])
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
938
|
+
# One file per turn: each turn is its own process, so nothing can be filed
|
|
939
|
+
# under the wrong turn.
|
|
940
|
+
folder = tempfile.mkdtemp(prefix="preman-tools-")
|
|
941
|
+
events_path = os.path.join(folder, "tool_events.jsonl")
|
|
942
|
+
try:
|
|
943
|
+
done = subprocess.run(
|
|
944
|
+
argv,
|
|
945
|
+
input=_prompt(message, history),
|
|
946
|
+
capture_output=True,
|
|
947
|
+
text=True,
|
|
948
|
+
cwd=str(_REPO),
|
|
949
|
+
timeout=_TIMEOUT,
|
|
950
|
+
env=dict(os.environ, PREMAN_TOOL_EVENTS_FILE=events_path),
|
|
951
|
+
)
|
|
952
|
+
if done.returncode != 0:
|
|
953
|
+
tail = (done.stderr or "").strip()[-600:]
|
|
954
|
+
raise RuntimeError("%s exited %s: %s" % (_ARGV[0], done.returncode, tail))
|
|
955
|
+
events, capture = _read_events(events_path)
|
|
956
|
+
finally:
|
|
957
|
+
shutil.rmtree(folder, ignore_errors=True)
|
|
958
|
+
return done.stdout.strip(), events, capture
|
|
547
959
|
`;
|
|
548
960
|
|
|
549
961
|
// ── Assembly ────────────────────────────────────────────────────────────
|
|
@@ -596,7 +1008,7 @@ export function buildWrapper(found, { tree = [] } = {}) {
|
|
|
596
1008
|
|
|
597
1009
|
if (shape === "cli_subprocess") {
|
|
598
1010
|
return {
|
|
599
|
-
shim: header + CLI_SUBPROCESS.replace("%%COMMAND%%", litList(command)) + PY_NORMALISE,
|
|
1011
|
+
shim: header + CLI_SUBPROCESS.replace("%%COMMAND%%", litList(command)) + PY_NORMALISE + PY_CHAT_REPORTED,
|
|
600
1012
|
bridge: "",
|
|
601
1013
|
};
|
|
602
1014
|
}
|
|
@@ -615,7 +1027,8 @@ export function buildWrapper(found, { tree = [] } = {}) {
|
|
|
615
1027
|
+ NODE_BRIDGE
|
|
616
1028
|
.replace("%%NODE_RUNTIME%%", lit(runtime))
|
|
617
1029
|
.replace("%%ENTRY_SYMBOL%%", lit(symbol))
|
|
618
|
-
+ PY_NORMALISE
|
|
1030
|
+
+ PY_NORMALISE
|
|
1031
|
+
+ PY_CHAT;
|
|
619
1032
|
const bridge = NODE_SIDE
|
|
620
1033
|
.replace(/%%NODE_ENTRY%%/g, lit("../" + nodeEntry))
|
|
621
1034
|
.replace(/%%ENTRY_SYMBOL%%/g, lit(symbol));
|
|
@@ -631,12 +1044,20 @@ export function buildWrapper(found, { tree = [] } = {}) {
|
|
|
631
1044
|
turn = turn.replace("%%METHOD%%", lit(method));
|
|
632
1045
|
}
|
|
633
1046
|
|
|
634
|
-
const
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
1047
|
+
const loader = PY_LOADER
|
|
1048
|
+
.replace("%%ENTRY_FILE%%", lit(entryFile))
|
|
1049
|
+
.replace("%%ENTRY_SYMBOL%%", lit(symbol));
|
|
1050
|
+
|
|
1051
|
+
// langgraph reads tool calls out of the graph's own messages (§5.1 of the
|
|
1052
|
+
// design), not by wrapping functions, so it keeps the plain `chat()`.
|
|
1053
|
+
if (shape === "langgraph") {
|
|
1054
|
+
return { shim: header + loader + turn + PY_NORMALISE + PY_CHAT, bridge: "" };
|
|
1055
|
+
}
|
|
1056
|
+
// Checked only for the shapes that use it, so a tool path this shape never
|
|
1057
|
+
// reads cannot fail the build. Every value is checked before it is quoted
|
|
1058
|
+
// into the template, so a tool value a model invented cannot interpolate.
|
|
1059
|
+
const tools = toolsLiteral(found?.tools);
|
|
1060
|
+
const shim = header + loader + PY_CAPTURE.replace("%%TOOLS%%", tools) + turn + PY_NORMALISE + PY_CHAT_CAPTURING;
|
|
640
1061
|
return { shim, bridge: "" };
|
|
641
1062
|
}
|
|
642
1063
|
|
|
@@ -660,6 +1081,9 @@ export function buildFreeform(found, { tree = [], turnSource = "" } = {}) {
|
|
|
660
1081
|
}
|
|
661
1082
|
const body = String(turnSource || "").trimEnd();
|
|
662
1083
|
if (!body.trim()) throw new TemplateError("no conversion body was returned");
|
|
1084
|
+
// Computed before any text is built, so a tool value a model invented does
|
|
1085
|
+
// not get the chance to interpolate into a template.
|
|
1086
|
+
const tools = toolsLiteral(found?.tools);
|
|
663
1087
|
|
|
664
1088
|
const header = HEADER
|
|
665
1089
|
.replace("%%SHAPE%%", "other")
|
|
@@ -676,9 +1100,11 @@ export function buildFreeform(found, { tree = [], turnSource = "" } = {}) {
|
|
|
676
1100
|
+ PY_LOADER
|
|
677
1101
|
.replace("%%ENTRY_FILE%%", lit(entryFile))
|
|
678
1102
|
.replace("%%ENTRY_SYMBOL%%", lit(symbol))
|
|
1103
|
+
+ PY_CAPTURE.replace("%%TOOLS%%", tools)
|
|
679
1104
|
+ "\n\ndef _turn(message, history):\n"
|
|
680
1105
|
+ ` ${symbol} = _target()\n`
|
|
681
1106
|
+ body + "\n"
|
|
682
|
-
+ PY_NORMALISE
|
|
1107
|
+
+ PY_NORMALISE
|
|
1108
|
+
+ PY_CHAT_CAPTURING;
|
|
683
1109
|
return { shim, bridge: "" };
|
|
684
1110
|
}
|
package/bin/hook.js
CHANGED
|
@@ -190,6 +190,11 @@ export function hookIsBehind(invocation) {
|
|
|
190
190
|
return isOlderRelease(pinned, running);
|
|
191
191
|
}
|
|
192
192
|
|
|
193
|
+
/** A path as one POSIX shell word, safe for any character a home directory allows. */
|
|
194
|
+
function shellSingleQuote(value) {
|
|
195
|
+
return `'${String(value).replace(/'/g, `'\\''`)}'`;
|
|
196
|
+
}
|
|
197
|
+
|
|
193
198
|
function hookBody(invocation) {
|
|
194
199
|
// `exec` is deliberately absent: we want the wrapper to survive the CLI exiting
|
|
195
200
|
// non-zero and still exit 0 itself.
|
|
@@ -201,14 +206,29 @@ ${MARKER}
|
|
|
201
206
|
# Managed by PreMan. Runs the local endpoint suite before the push leaves.
|
|
202
207
|
# Advisory unless this repository opted into blocking: only exit code
|
|
203
208
|
# ${BLOCK_EXIT_CODE} stops a push, so a crash or a timeout still lets it through.
|
|
209
|
+
#
|
|
210
|
+
# PREMAN_HOOK_BIN is the bin directory PreMan was installed in when this hook
|
|
211
|
+
# was written. A hook is generated once and then runs for months, and a version
|
|
212
|
+
# manager switching Node takes that whole directory off PATH -- after which
|
|
213
|
+
# this hook printed "not on PATH, checks skipped" on every single push, forever
|
|
214
|
+
# and silently, which is the worst available outcome for a tool whose only job
|
|
215
|
+
# is checking pushes. git does not run a login shell, so nvm's own PATH setup
|
|
216
|
+
# is not here to recover it either. Recording the directory costs nothing and
|
|
217
|
+
# makes the second attempt the one that works.
|
|
204
218
|
if [ -z "\${PREMAN_SKIP_HOOK}" ]; then
|
|
219
|
+
PREMAN_HOOK_BIN=${shellSingleQuote(path.dirname(process.execPath))}
|
|
205
220
|
PREMAN_HOOK=1 ${invocation} verify --pre-push --timeout ${HOOK_TIMEOUT_SECONDS}
|
|
206
221
|
preman_status=$?
|
|
222
|
+
if [ "$preman_status" -eq 127 ] && [ -d "$PREMAN_HOOK_BIN" ]; then
|
|
223
|
+
PATH="$PREMAN_HOOK_BIN:$PATH" PREMAN_HOOK=1 ${invocation} verify --pre-push --timeout ${HOOK_TIMEOUT_SECONDS}
|
|
224
|
+
preman_status=$?
|
|
225
|
+
fi
|
|
207
226
|
if [ "$preman_status" -eq ${BLOCK_EXIT_CODE} ]; then
|
|
208
227
|
exit ${BLOCK_EXIT_CODE}
|
|
209
228
|
fi
|
|
210
229
|
if [ "$preman_status" -eq 127 ]; then
|
|
211
|
-
printf '[preman] the PreMan CLI is not on PATH
|
|
230
|
+
printf '[preman] the PreMan CLI is not on PATH, so push checks were skipped.\\n' >&2
|
|
231
|
+
printf '[preman] run: preman hook install --force\\n' >&2
|
|
212
232
|
elif [ "$preman_status" -ne 0 ]; then
|
|
213
233
|
printf '[preman] checks skipped (advisory, exit %s)\\n' "$preman_status" >&2
|
|
214
234
|
fi
|
package/bin/integrations.js
CHANGED
|
@@ -587,13 +587,19 @@ async function showDesktopSignedIn(openDesktopSignedIn, args, creds, stopOpening
|
|
|
587
587
|
`PreMan was already open and kept its previous sign-in.\n` +
|
|
588
588
|
` Quit and reopen it to switch${email ? ` to ${email}` : ""}.\n`
|
|
589
589
|
);
|
|
590
|
+
} else if (opened.state === "launch-failed") {
|
|
591
|
+
// The installer has already printed what open said and what to do about it,
|
|
592
|
+
// so this step adds nothing by repeating it -- but it must not fall through
|
|
593
|
+
// to the line below, which claims the app is up. It said "Opened PreMan" on
|
|
594
|
+
// top of a Gatekeeper refusal, and then the walk signed off with a line
|
|
595
|
+
// about the app being on screen.
|
|
590
596
|
} else {
|
|
591
597
|
// Either the session could not be handed over or this app is too old to take
|
|
592
598
|
// it up. Say so rather than let the customer wonder why they are looking at
|
|
593
599
|
// a login screen.
|
|
594
600
|
process.stdout.write("Opened PreMan \u2014 sign in with the account you just used.\n");
|
|
595
601
|
}
|
|
596
|
-
return opened.state !== "not-installed";
|
|
602
|
+
return opened.state !== "not-installed" && opened.state !== "launch-failed";
|
|
597
603
|
}
|
|
598
604
|
|
|
599
605
|
/** One step heading, in the one format the whole walk uses. */
|
|
@@ -869,7 +875,11 @@ export async function onboardCommand(
|
|
|
869
875
|
"Not a git repository, so there is nothing to hook here.\n" +
|
|
870
876
|
`Run '${cliInvocation()} hook install' inside your API repo.\n`
|
|
871
877
|
);
|
|
872
|
-
|
|
878
|
+
// Returning nothing means "done" to the summary, so the walk that
|
|
879
|
+
// had just explained it could not install a hook signed off with a
|
|
880
|
+
// tick against Push checks five lines later. The last line of a walk
|
|
881
|
+
// is the one people keep, and it was the one saying the opposite.
|
|
882
|
+
return { state: "skipped", detail: "not a git repository" };
|
|
873
883
|
}
|
|
874
884
|
throw err;
|
|
875
885
|
}
|
|
@@ -878,7 +888,7 @@ export async function onboardCommand(
|
|
|
878
888
|
// nothing would leave the customer believing their pushes are checked.
|
|
879
889
|
if (result.action === "conflict" || result.action === "unproven") {
|
|
880
890
|
process.stdout.write(`Not installed: ${result.detail}\n ${result.path}\n`);
|
|
881
|
-
return;
|
|
891
|
+
return { state: "skipped", detail: result.detail };
|
|
882
892
|
}
|
|
883
893
|
|
|
884
894
|
// One line, like every other step. The pinned invocation and the skip
|
|
@@ -1020,7 +1030,13 @@ export async function onboardCommand(
|
|
|
1020
1030
|
const result = outcome.get(step.name);
|
|
1021
1031
|
if (!result) continue;
|
|
1022
1032
|
if (result.state === "done") process.stdout.write(`${MARK.ok()} ${step.name}\n`);
|
|
1023
|
-
else if (result.state === "skipped")
|
|
1033
|
+
else if (result.state === "skipped") {
|
|
1034
|
+
// The reason belongs on the line. "(skipped)" against a step the customer
|
|
1035
|
+
// said yes to reads as a bug in the walk rather than a condition on their
|
|
1036
|
+
// machine, and the condition is usually the thing they can act on.
|
|
1037
|
+
const why = result.detail ? ` (${result.detail})` : " (skipped)";
|
|
1038
|
+
process.stdout.write(`${MARK.skip()} ${step.name}${why}\n`);
|
|
1039
|
+
}
|
|
1024
1040
|
else process.stdout.write(`${MARK.fail()} ${step.name}: ${result.detail}\n`);
|
|
1025
1041
|
}
|
|
1026
1042
|
// The app is on screen by the time this prints, so telling them to go and open
|