premanmcp 1.1.14 → 1.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/decorate_tools.py +368 -0
- package/bin/eval.js +3 -0
- package/bin/eval_entry.js +134 -6
- package/bin/eval_harness.py +12 -0
- package/bin/eval_target.js +13 -8
- package/bin/eval_templates.js +238 -349
- package/bin/preman/__init__.py +267 -0
- package/package.json +2 -2
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""Plan the edit that puts @preman_tool on an agent's tool functions.
|
|
2
|
+
|
|
3
|
+
Run by `preman eval target` as
|
|
4
|
+
|
|
5
|
+
uv run --no-project --with libcst python decorate_tools.py --repo ROOT --plan PLAN.json
|
|
6
|
+
|
|
7
|
+
PLAN.json is {"tools": [{"file", "symbol", "tool_name"}]}. Prints one JSON
|
|
8
|
+
object on stdout and never writes a file: the CLI shows the diff, asks, and
|
|
9
|
+
writes. The same plan on the same files always gives the same bytes
|
|
10
|
+
(docs/superpowers/specs/2026-10-04-tool-decorators-design.md §6.1).
|
|
11
|
+
|
|
12
|
+
Nothing is added to the customer's dependencies: each edited file gets a
|
|
13
|
+
guarded import whose fallback decorates with nothing, so the code runs
|
|
14
|
+
unchanged wherever PreMan's recorder is not importable (everywhere but an eval).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import difflib
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import sys
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
import libcst as cst
|
|
28
|
+
import libcst.matchers as m
|
|
29
|
+
|
|
30
|
+
SKIP_DIRS = {
|
|
31
|
+
"node_modules", ".venv", "venv", ".git", "dist", "build", "__pycache__",
|
|
32
|
+
".next", ".turbo", "target", ".mypy_cache", ".pytest_cache", ".preman",
|
|
33
|
+
}
|
|
34
|
+
NAME_SUFFIXES = {".py", ".json", ".yaml", ".yml", ".toml"}
|
|
35
|
+
MAX_SCAN_BYTES = 1_000_000
|
|
36
|
+
|
|
37
|
+
GUARD = (
|
|
38
|
+
"try:\n"
|
|
39
|
+
" from preman import tool as preman_tool\n"
|
|
40
|
+
"except ImportError: # PreMan's eval recorder; without it, decorate with nothing\n"
|
|
41
|
+
" def preman_tool(name):\n"
|
|
42
|
+
" return lambda fn: fn\n"
|
|
43
|
+
)
|
|
44
|
+
GUARD_IMPORT = "from preman import tool as preman_tool"
|
|
45
|
+
|
|
46
|
+
# Framework tool decorators whose tracing plugin, installed for the eval,
|
|
47
|
+
# already records every call. Decorating them too counts each call twice
|
|
48
|
+
# (spec §4.4), so these are left alone and recorded by the plugin.
|
|
49
|
+
COVERED = {
|
|
50
|
+
("langchain_core.tools", "tool"): "langchain",
|
|
51
|
+
("langchain.tools", "tool"): "langchain",
|
|
52
|
+
("langchain.agents", "tool"): "langchain",
|
|
53
|
+
}
|
|
54
|
+
# Framework tools whose plugin may or may not record them: unverified, so
|
|
55
|
+
# neither decorated (possible double count) nor trusted to the plugin.
|
|
56
|
+
UNSUPPORTED = {
|
|
57
|
+
("agents", "function_tool"): "OpenAI Agents SDK tools are not supported yet",
|
|
58
|
+
}
|
|
59
|
+
# Wrappers that call the function they wrap and return a plain function.
|
|
60
|
+
# @preman_tool may sit above these. Every other decorator may register the
|
|
61
|
+
# function it receives (@agent.tool, @mcp.tool()) or turn it into an object
|
|
62
|
+
# (@property, CrewAI's @tool), so @preman_tool goes beneath the lowest of
|
|
63
|
+
# those: what the framework holds is then the recorded function.
|
|
64
|
+
PASS_THROUGH = {
|
|
65
|
+
("functools", "lru_cache"), ("functools", "cache"), ("tenacity", "retry"),
|
|
66
|
+
("backoff", "on_exception"), ("backoff", "on_predicate"),
|
|
67
|
+
}
|
|
68
|
+
# What an existing PreMan decorator resolves to: the guard's name, or a
|
|
69
|
+
# hand-written `import preman` + `@preman.tool`.
|
|
70
|
+
PREMAN = {("", "preman_tool"), ("preman", "tool")}
|
|
71
|
+
# Valid tool names: alphanumeric, hyphen, underscore, 1-64 chars.
|
|
72
|
+
TOOL_NAME_PATTERN = re.compile(r"[A-Za-z0-9_-]{1,64}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _dotted(node) -> str:
|
|
76
|
+
if isinstance(node, cst.Name):
|
|
77
|
+
return node.value
|
|
78
|
+
if isinstance(node, cst.Attribute):
|
|
79
|
+
head = _dotted(node.value)
|
|
80
|
+
return "%s.%s" % (head, node.attr.value) if head else ""
|
|
81
|
+
return ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _imports(module: cst.Module) -> dict[str, str]:
|
|
85
|
+
"""Local name -> what it refers to, e.g. {"tool": "langchain_core.tools.tool", "lt": "langchain_core.tools"}."""
|
|
86
|
+
names: dict[str, str] = {}
|
|
87
|
+
for stmt in module.body:
|
|
88
|
+
if not isinstance(stmt, cst.SimpleStatementLine):
|
|
89
|
+
continue
|
|
90
|
+
for small in stmt.body:
|
|
91
|
+
if isinstance(small, cst.ImportFrom) and small.module is not None and not small.relative:
|
|
92
|
+
if isinstance(small.names, cst.ImportStar):
|
|
93
|
+
continue
|
|
94
|
+
base = _dotted(small.module)
|
|
95
|
+
for alias in small.names:
|
|
96
|
+
local = alias.asname.name.value if alias.asname else _dotted(alias.name)
|
|
97
|
+
names[local] = "%s.%s" % (base, _dotted(alias.name))
|
|
98
|
+
elif isinstance(small, cst.Import):
|
|
99
|
+
for alias in small.names:
|
|
100
|
+
full = _dotted(alias.name)
|
|
101
|
+
local = alias.asname.name.value if alias.asname else full.split(".")[0]
|
|
102
|
+
names[local] = full if alias.asname else local
|
|
103
|
+
return names
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _resolve(expr, imports: dict[str, str]) -> tuple[str, str]:
|
|
107
|
+
"""(module, name) a decorator expression refers to; ("", name) for an unimported bare name."""
|
|
108
|
+
if isinstance(expr, cst.Call):
|
|
109
|
+
expr = expr.func
|
|
110
|
+
dotted = _dotted(expr)
|
|
111
|
+
if not dotted:
|
|
112
|
+
return "", ""
|
|
113
|
+
head, _, rest = dotted.partition(".")
|
|
114
|
+
full = imports.get(head, head)
|
|
115
|
+
full = "%s.%s" % (full, rest) if rest else full
|
|
116
|
+
module, _, name = full.rpartition(".")
|
|
117
|
+
return module, name
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _string_arg(expr) -> str:
|
|
121
|
+
if isinstance(expr, cst.Call) and expr.args and isinstance(expr.args[0].value, cst.SimpleString):
|
|
122
|
+
return expr.args[0].value.evaluated_value
|
|
123
|
+
return ""
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class _Yields(cst.CSTVisitor):
|
|
127
|
+
def __init__(self):
|
|
128
|
+
self.found = False
|
|
129
|
+
|
|
130
|
+
def visit_FunctionDef(self, node):
|
|
131
|
+
return False
|
|
132
|
+
|
|
133
|
+
def visit_Lambda(self, node):
|
|
134
|
+
return False
|
|
135
|
+
|
|
136
|
+
def visit_Yield(self, node):
|
|
137
|
+
self.found = True
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _is_generator(fn: cst.FunctionDef) -> bool:
|
|
141
|
+
visitor = _Yields()
|
|
142
|
+
fn.body.visit(visitor)
|
|
143
|
+
return visitor.found
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _from_function_refs(module: cst.Module) -> dict[str, tuple[str, str]]:
|
|
147
|
+
"""Functions handed to a framework's tool factory: name -> (kind, framework or reason)."""
|
|
148
|
+
refs: dict[str, tuple[str, str]] = {}
|
|
149
|
+
|
|
150
|
+
class Calls(cst.CSTVisitor):
|
|
151
|
+
def visit_Call(self, node):
|
|
152
|
+
if not isinstance(node.func, cst.Attribute):
|
|
153
|
+
return
|
|
154
|
+
attr = node.func.attr.value
|
|
155
|
+
for arg in node.args:
|
|
156
|
+
if not isinstance(arg.value, cst.Name):
|
|
157
|
+
continue
|
|
158
|
+
keyword = arg.keyword.value if arg.keyword else ""
|
|
159
|
+
if attr == "from_function" and keyword in ("", "func"):
|
|
160
|
+
refs.setdefault(arg.value.value, ("covered", "langchain"))
|
|
161
|
+
elif attr == "from_defaults" and keyword in ("", "fn"):
|
|
162
|
+
refs.setdefault(arg.value.value, ("dropped", "LlamaIndex tools are not supported yet"))
|
|
163
|
+
|
|
164
|
+
module.visit(Calls())
|
|
165
|
+
return refs
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class _Decorate(cst.CSTTransformer):
|
|
169
|
+
"""Adds @preman_tool to the wanted functions and records an outcome for each."""
|
|
170
|
+
|
|
171
|
+
def __init__(self, wanted, imports, factory_refs, name_ok):
|
|
172
|
+
self.wanted = wanted # {(class or "", function): tool_name}
|
|
173
|
+
self.imports = imports
|
|
174
|
+
self.factory_refs = factory_refs
|
|
175
|
+
self.name_ok = name_ok
|
|
176
|
+
self.classes: list[str] = []
|
|
177
|
+
self.depth = 0
|
|
178
|
+
self.outcomes: dict[tuple[str, str], tuple[str, str]] = {}
|
|
179
|
+
self.added = False
|
|
180
|
+
|
|
181
|
+
def _outcome(self, key, kind, detail):
|
|
182
|
+
# A symbol defined more than once (if/else, try/except, a redefinition)
|
|
183
|
+
# is edited at every definition and reported once; any definition that
|
|
184
|
+
# is not decorated decides the report.
|
|
185
|
+
if self.outcomes.get(key, ("decorate", ""))[0] == "decorate":
|
|
186
|
+
self.outcomes[key] = (kind, detail)
|
|
187
|
+
|
|
188
|
+
def visit_ClassDef(self, node):
|
|
189
|
+
self.classes.append(node.name.value)
|
|
190
|
+
|
|
191
|
+
def leave_ClassDef(self, original, updated):
|
|
192
|
+
self.classes.pop()
|
|
193
|
+
return updated
|
|
194
|
+
|
|
195
|
+
def visit_FunctionDef(self, node):
|
|
196
|
+
self.depth += 1
|
|
197
|
+
|
|
198
|
+
def leave_FunctionDef(self, original, updated):
|
|
199
|
+
self.depth -= 1
|
|
200
|
+
if self.depth != 0 or len(self.classes) > 1:
|
|
201
|
+
return updated
|
|
202
|
+
key = (self.classes[0] if self.classes else "", original.name.value)
|
|
203
|
+
if key not in self.wanted:
|
|
204
|
+
return updated
|
|
205
|
+
tool_name = self.wanted[key]
|
|
206
|
+
resolved = [_resolve(d.decorator, self.imports) for d in original.decorators]
|
|
207
|
+
for d, where in zip(original.decorators, resolved):
|
|
208
|
+
if where in PREMAN:
|
|
209
|
+
existing = _string_arg(d.decorator)
|
|
210
|
+
if existing == tool_name:
|
|
211
|
+
self._outcome(key, "decorate", "")
|
|
212
|
+
else:
|
|
213
|
+
self._outcome(key, "dropped", 'already decorated as @preman_tool("%s")' % existing)
|
|
214
|
+
return updated
|
|
215
|
+
for where in resolved:
|
|
216
|
+
if where in COVERED:
|
|
217
|
+
self._outcome(key, "covered", COVERED[where])
|
|
218
|
+
return updated
|
|
219
|
+
if where in UNSUPPORTED:
|
|
220
|
+
self._outcome(key, "dropped", UNSUPPORTED[where])
|
|
221
|
+
return updated
|
|
222
|
+
if not key[0] and key[1] in self.factory_refs:
|
|
223
|
+
self._outcome(key, *self.factory_refs[key[1]])
|
|
224
|
+
return updated
|
|
225
|
+
if _is_generator(original):
|
|
226
|
+
self._outcome(key, "dropped", "a generator function, which preman_tool cannot record")
|
|
227
|
+
return updated
|
|
228
|
+
if not self.name_ok(tool_name):
|
|
229
|
+
self._outcome(key, "dropped", 'the tool name "%s" was not found as a string anywhere in the repository' % tool_name)
|
|
230
|
+
return updated
|
|
231
|
+
deco = cst.Decorator(decorator=cst.parse_expression("preman_tool(%s)" % json.dumps(tool_name)))
|
|
232
|
+
at = 0
|
|
233
|
+
for i, where in enumerate(resolved):
|
|
234
|
+
if where not in PASS_THROUGH:
|
|
235
|
+
at = i + 1
|
|
236
|
+
decorators = list(updated.decorators)
|
|
237
|
+
decorators.insert(at, deco)
|
|
238
|
+
self._outcome(key, "decorate", "")
|
|
239
|
+
self.added = True
|
|
240
|
+
return updated.with_changes(decorators=decorators)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _has_guard(module: cst.Module) -> bool:
|
|
244
|
+
return any(
|
|
245
|
+
isinstance(stmt, cst.Try) and GUARD_IMPORT in module.code_for_node(stmt)
|
|
246
|
+
for stmt in module.body
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _with_guard(module: cst.Module) -> cst.Module:
|
|
251
|
+
body = list(module.body)
|
|
252
|
+
i = 0
|
|
253
|
+
if body and m.matches(body[0], m.SimpleStatementLine(body=[m.Expr(m.SimpleString() | m.ConcatenatedString())])):
|
|
254
|
+
i = 1
|
|
255
|
+
while i < len(body) and m.matches(body[i], m.SimpleStatementLine(body=[m.ImportFrom(module=m.Name("__future__"))])):
|
|
256
|
+
i += 1
|
|
257
|
+
if i < len(body) and not body[i].leading_lines:
|
|
258
|
+
body[i] = body[i].with_changes(leading_lines=[cst.EmptyLine()])
|
|
259
|
+
body.insert(i, cst.parse_module(GUARD).body[0])
|
|
260
|
+
return module.with_changes(body=body)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _name_blob(repo: Path) -> str:
|
|
264
|
+
parts = []
|
|
265
|
+
for dirpath, dirnames, filenames in os.walk(repo):
|
|
266
|
+
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS and not d.startswith("."))
|
|
267
|
+
for name in sorted(filenames):
|
|
268
|
+
path = Path(dirpath) / name
|
|
269
|
+
if path.suffix not in NAME_SUFFIXES:
|
|
270
|
+
continue
|
|
271
|
+
try:
|
|
272
|
+
if path.stat().st_size <= MAX_SCAN_BYTES:
|
|
273
|
+
parts.append(path.read_text(encoding="utf-8", errors="ignore"))
|
|
274
|
+
except OSError:
|
|
275
|
+
continue
|
|
276
|
+
return "\n".join(parts)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def plan(repo: Path, tools: list[dict]) -> dict:
|
|
280
|
+
repo_resolved = repo.resolve()
|
|
281
|
+
by_file: dict[str, list[dict]] = {}
|
|
282
|
+
decorate, covered, dropped, files = [], [], [], []
|
|
283
|
+
blob: list[str] = []
|
|
284
|
+
# Validate tool_names first and separate valid from invalid
|
|
285
|
+
for t in tools:
|
|
286
|
+
name = t.get("tool_name")
|
|
287
|
+
if not isinstance(name, str) or not TOOL_NAME_PATTERN.fullmatch(name):
|
|
288
|
+
dropped.append({**t, "why": 'the tool name "%s" is not a tool name' % name})
|
|
289
|
+
else:
|
|
290
|
+
by_file.setdefault(str(t["file"]), []).append(t)
|
|
291
|
+
|
|
292
|
+
def name_ok(name: str) -> bool:
|
|
293
|
+
if not blob:
|
|
294
|
+
blob.append(_name_blob(repo))
|
|
295
|
+
return '"%s"' % name in blob[0] or "'%s'" % name in blob[0]
|
|
296
|
+
|
|
297
|
+
for file in sorted(by_file):
|
|
298
|
+
entries = sorted(by_file[file], key=lambda t: t["symbol"])
|
|
299
|
+
# Check for absolute paths, ".." segments, and ensure the file is inside the repo
|
|
300
|
+
file_path_obj = Path(file)
|
|
301
|
+
if file_path_obj.is_absolute() or ".." in file_path_obj.parts:
|
|
302
|
+
dropped += [{**t, "why": "%s is outside the repository" % file} for t in entries]
|
|
303
|
+
continue
|
|
304
|
+
path = (repo_resolved / file).resolve()
|
|
305
|
+
if not path.is_relative_to(repo_resolved):
|
|
306
|
+
dropped += [{**t, "why": "%s is outside the repository" % file} for t in entries]
|
|
307
|
+
continue
|
|
308
|
+
if not path.is_file():
|
|
309
|
+
dropped += [{**t, "why": "%s does not exist" % file} for t in entries]
|
|
310
|
+
continue
|
|
311
|
+
try:
|
|
312
|
+
with open(path, encoding="utf-8", newline="") as f:
|
|
313
|
+
before = f.read()
|
|
314
|
+
except UnicodeDecodeError:
|
|
315
|
+
dropped += [{**t, "why": "%s is not valid UTF-8" % file} for t in entries]
|
|
316
|
+
continue
|
|
317
|
+
try:
|
|
318
|
+
module = cst.parse_module(before)
|
|
319
|
+
except cst.ParserSyntaxError:
|
|
320
|
+
dropped += [{**t, "why": "%s does not parse as Python" % file} for t in entries]
|
|
321
|
+
continue
|
|
322
|
+
wanted = {}
|
|
323
|
+
for t in entries:
|
|
324
|
+
cls, _, fn = t["symbol"].rpartition(".")
|
|
325
|
+
wanted[(cls, fn)] = t["tool_name"]
|
|
326
|
+
transformer = _Decorate(wanted, _imports(module), _from_function_refs(module), name_ok)
|
|
327
|
+
edited = module.visit(transformer)
|
|
328
|
+
if transformer.added and not _has_guard(edited):
|
|
329
|
+
edited = _with_guard(edited)
|
|
330
|
+
for t in entries:
|
|
331
|
+
cls, _, fn = t["symbol"].rpartition(".")
|
|
332
|
+
kind, detail = transformer.outcomes.get((cls, fn), ("dropped", "%s is not defined in %s" % (t["symbol"], file)))
|
|
333
|
+
if kind == "decorate":
|
|
334
|
+
decorate.append(dict(t))
|
|
335
|
+
elif kind == "covered":
|
|
336
|
+
covered.append({**t, "covered_by": detail})
|
|
337
|
+
else:
|
|
338
|
+
dropped.append({**t, "why": detail})
|
|
339
|
+
if edited.code != before:
|
|
340
|
+
files.append({"path": file, "before": before, "after": edited.code})
|
|
341
|
+
|
|
342
|
+
diff = "".join(
|
|
343
|
+
"".join(difflib.unified_diff(
|
|
344
|
+
f["before"].splitlines(True), f["after"].splitlines(True),
|
|
345
|
+
"a/%s" % f["path"], "b/%s" % f["path"],
|
|
346
|
+
))
|
|
347
|
+
for f in files
|
|
348
|
+
)
|
|
349
|
+
return {"ok": True, "files": files, "diff": diff, "decorate": decorate,
|
|
350
|
+
"covered": covered, "dropped": dropped}
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def main(argv: list[str]) -> int:
|
|
354
|
+
parser = argparse.ArgumentParser()
|
|
355
|
+
parser.add_argument("--repo", required=True)
|
|
356
|
+
parser.add_argument("--plan", required=True)
|
|
357
|
+
args = parser.parse_args(argv)
|
|
358
|
+
try:
|
|
359
|
+
spec = json.loads(Path(args.plan).read_text(encoding="utf-8"))
|
|
360
|
+
result = plan(Path(args.repo).resolve(), list(spec.get("tools") or []))
|
|
361
|
+
except (ValueError, KeyError, OSError, TypeError) as exc:
|
|
362
|
+
result = {"ok": False, "why": "%s: %s" % (type(exc).__name__, exc)}
|
|
363
|
+
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
|
364
|
+
return 0
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
if __name__ == "__main__":
|
|
368
|
+
sys.exit(main(sys.argv[1:]))
|
package/bin/eval.js
CHANGED
|
@@ -966,6 +966,9 @@ export function entryImportFailure(python, sysPath, { timeoutMs = 120_000, log =
|
|
|
966
966
|
|
|
967
967
|
const snippet = [
|
|
968
968
|
`import sys`,
|
|
969
|
+
// The CLI's own bin/, where PreMan's recorder (`preman`) lives, as the
|
|
970
|
+
// harness puts it on the path for itself.
|
|
971
|
+
`sys.path.insert(0,${JSON.stringify(path.dirname(HARNESS_PATH))})`,
|
|
969
972
|
`sys.path.insert(0,${JSON.stringify(sysPath)})`,
|
|
970
973
|
`import ${CALLABLE_MODULE} as _w`,
|
|
971
974
|
`_c=getattr(_w,"preman_selfcheck",None)`,
|
package/bin/eval_entry.js
CHANGED
|
@@ -22,9 +22,11 @@
|
|
|
22
22
|
|
|
23
23
|
import { spawnSync } from "node:child_process";
|
|
24
24
|
import { createHash } from "node:crypto";
|
|
25
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
25
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
26
|
+
import os from "node:os";
|
|
26
27
|
import path from "node:path";
|
|
27
28
|
import { createInterface } from "node:readline";
|
|
29
|
+
import { fileURLToPath } from "node:url";
|
|
28
30
|
|
|
29
31
|
import { callBackendJson, cliInvocation, resolveApiKey } from "./shared.js";
|
|
30
32
|
import { REPO_CONFIG, findConfigRoot, findRepoRoot, saveEntry } from "./target.js";
|
|
@@ -553,7 +555,7 @@ function ignoreEntryDir(root) {
|
|
|
553
555
|
return true;
|
|
554
556
|
}
|
|
555
557
|
|
|
556
|
-
function record(found, file, root, source, { language, alsoWrote = [] }) {
|
|
558
|
+
function record(found, file, root, source, { language, alsoWrote = [], edited = [] }) {
|
|
557
559
|
const sha256 = createHash("sha256").update(source).digest("hex");
|
|
558
560
|
// Hashed from disk rather than from what was generated: the point of the
|
|
559
561
|
// record is to notice a later edit, and a person offered `[e]dit it first`
|
|
@@ -566,12 +568,21 @@ function record(found, file, root, source, { language, alsoWrote = [] }) {
|
|
|
566
568
|
sha256: createHash("sha256").update(readFileSync(extra, "utf8")).digest("hex"),
|
|
567
569
|
});
|
|
568
570
|
}
|
|
571
|
+
// The customer's own files that carry @preman_tool. Fingerprinted so an edit
|
|
572
|
+
// to them sends the next run back through review: a tool added or renamed
|
|
573
|
+
// since approval would otherwise run undecorated and read as zero calls.
|
|
574
|
+
for (const rel of edited) {
|
|
575
|
+
const full = path.join(root, rel);
|
|
576
|
+
if (!existsSync(full)) continue;
|
|
577
|
+
files.push({ path: rel, sha256: createHash("sha256").update(readFileSync(full, "utf8")).digest("hex") });
|
|
578
|
+
}
|
|
569
579
|
const entry = {
|
|
570
580
|
file: path.relative(root, file).split(path.sep).join("/"),
|
|
571
581
|
ref: ENTRY_REF,
|
|
572
582
|
sysPath: ENTRY_DIR,
|
|
573
583
|
sha256,
|
|
574
584
|
files,
|
|
585
|
+
tools: Array.isArray(found.tools) ? found.tools : [],
|
|
575
586
|
// How the wrapper was built. Recorded because it is the one fact that
|
|
576
587
|
// explains the file's contents later: "this came from the langgraph
|
|
577
588
|
// template" is a different thing to debug than "a model wrote this".
|
|
@@ -616,7 +627,10 @@ export async function searchForEntry({ projectPath, args, log = () => {} }) {
|
|
|
616
627
|
);
|
|
617
628
|
}
|
|
618
629
|
|
|
619
|
-
|
|
630
|
+
const decorated = await applyDecorations(found, { root, log });
|
|
631
|
+
return installShim(decorated.found, wrapperFor(decorated.found, tree), {
|
|
632
|
+
projectPath, args, log, edited: decorated.edited,
|
|
633
|
+
});
|
|
620
634
|
}
|
|
621
635
|
|
|
622
636
|
/**
|
|
@@ -683,7 +697,10 @@ export async function entryFromNamedSymbol(named, { projectPath, args, log = ()
|
|
|
683
697
|
(found?.reasoning ? `\n\nIt said: ${found.reasoning}` : "")
|
|
684
698
|
);
|
|
685
699
|
}
|
|
686
|
-
|
|
700
|
+
const decorated = await applyDecorations(found, { root, log });
|
|
701
|
+
return installShim(decorated.found, wrapperFor(decorated.found, [rel]), {
|
|
702
|
+
projectPath, args, log, edited: decorated.edited,
|
|
703
|
+
});
|
|
687
704
|
}
|
|
688
705
|
|
|
689
706
|
/**
|
|
@@ -727,7 +744,118 @@ function wrapperFor(found, tree) {
|
|
|
727
744
|
}
|
|
728
745
|
}
|
|
729
746
|
|
|
730
|
-
|
|
747
|
+
// ── Adding @preman_tool ─────────────────────────────────────────────────
|
|
748
|
+
|
|
749
|
+
const DECORATE_SCRIPT = fileURLToPath(new URL("./decorate_tools.py", import.meta.url));
|
|
750
|
+
const DECORATE_TIMEOUT_MS = 300_000; // a cold machine downloads libcst
|
|
751
|
+
/** Shapes whose wrapper reads @preman_tool (PY_DECORATED in eval_templates.js). */
|
|
752
|
+
const DECORATED_SHAPES = new Set(["py_history", "py_plain", "py_class", "other"]);
|
|
753
|
+
|
|
754
|
+
function planDecorations(found, root) {
|
|
755
|
+
const dir = mkdtempSync(path.join(os.tmpdir(), "preman-decorate-"));
|
|
756
|
+
try {
|
|
757
|
+
const planFile = path.join(dir, "plan.json");
|
|
758
|
+
writeFileSync(planFile, JSON.stringify({ tools: found.tools || [] }), "utf8");
|
|
759
|
+
const ran = spawnSync(
|
|
760
|
+
"uv",
|
|
761
|
+
["run", "--no-project", "--with", "libcst", "python", DECORATE_SCRIPT, "--repo", root, "--plan", planFile],
|
|
762
|
+
{ encoding: "utf8", timeout: DECORATE_TIMEOUT_MS }
|
|
763
|
+
);
|
|
764
|
+
if (ran.error?.code === "ENOENT") {
|
|
765
|
+
throw new TargetError(
|
|
766
|
+
"PreMan needs uv to add its tool decorators. Install it (https://docs.astral.sh/uv/) and re-run `preman eval target`."
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
const line = String(ran.stdout || "").trim().split("\n").pop() || "";
|
|
770
|
+
let out;
|
|
771
|
+
try {
|
|
772
|
+
out = JSON.parse(line);
|
|
773
|
+
} catch {
|
|
774
|
+
throw new TargetError(`PreMan could not plan its tool decorators: ${plannerFailure(ran)}`);
|
|
775
|
+
}
|
|
776
|
+
if (!out.ok) throw new TargetError(`PreMan could not plan its tool decorators: ${out.why}`);
|
|
777
|
+
return out;
|
|
778
|
+
} finally {
|
|
779
|
+
rmSync(dir, { recursive: true, force: true });
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/** Why the planner printed no plan: how it ended, then the end of what it said. */
|
|
784
|
+
function plannerFailure(ran) {
|
|
785
|
+
const cause =
|
|
786
|
+
ran.error?.code === "ETIMEDOUT"
|
|
787
|
+
? `it did not finish within ${DECORATE_TIMEOUT_MS / 60_000} minutes`
|
|
788
|
+
: ran.signal
|
|
789
|
+
? `it was stopped by ${ran.signal}`
|
|
790
|
+
: ran.error
|
|
791
|
+
? ran.error.message
|
|
792
|
+
: ran.status
|
|
793
|
+
? `it exited with status ${ran.status}`
|
|
794
|
+
: "it printed no plan";
|
|
795
|
+
const tail = (ran.stderr || "").trim().slice(-400);
|
|
796
|
+
return tail ? `${cause}: ${tail}` : cause;
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
/**
|
|
800
|
+
* Put @preman_tool on the agent's tools, with the customer's yes. Returns the
|
|
801
|
+
* answer with `tools` replaced by what the wrapper should watch, and the
|
|
802
|
+
* repository-relative files that now carry decorators.
|
|
803
|
+
*
|
|
804
|
+
* Nothing is installed: the block the edit writes falls back to a decorator
|
|
805
|
+
* that does nothing wherever PreMan's recorder is not importable, which is
|
|
806
|
+
* everywhere but an eval. Declining is allowed: the eval still runs and every
|
|
807
|
+
* turn says tool calls were not recorded.
|
|
808
|
+
*/
|
|
809
|
+
async function applyDecorations(found, { root, log }) {
|
|
810
|
+
if (!DECORATED_SHAPES.has(String(found.shape || "")) || !(found.tools || []).length) {
|
|
811
|
+
return { found, edited: [] };
|
|
812
|
+
}
|
|
813
|
+
const plan = planDecorations(found, root);
|
|
814
|
+
for (const d of plan.dropped) log(` not decorated: ${d.file}:${d.symbol} (${d.why})`);
|
|
815
|
+
for (const c of plan.covered) log(` recorded by the ${c.covered_by} tracing plugin, not decorated: ${c.file}:${c.symbol}`);
|
|
816
|
+
const watched = [...plan.decorate, ...plan.covered];
|
|
817
|
+
const edited = [...new Set(plan.decorate.map((d) => d.file))];
|
|
818
|
+
if (!plan.files.length) return { found: { ...found, tools: watched }, edited };
|
|
819
|
+
|
|
820
|
+
log("");
|
|
821
|
+
log("To record this agent's tool calls, PreMan would make these changes to your code:");
|
|
822
|
+
log("");
|
|
823
|
+
log(plan.diff);
|
|
824
|
+
log(
|
|
825
|
+
"@preman_tool records only while a PreMan eval is running. Everywhere else it leaves " +
|
|
826
|
+
"your function unchanged, and nothing needs installing."
|
|
827
|
+
);
|
|
828
|
+
const answer = await ask("Make these changes? [y]es / [n]o: ");
|
|
829
|
+
if (!answer.startsWith("y")) {
|
|
830
|
+
log("No changes made. The eval will run, but its tool calls will not be recorded.");
|
|
831
|
+
return {
|
|
832
|
+
found: {
|
|
833
|
+
...found,
|
|
834
|
+
tools: plan.covered,
|
|
835
|
+
tools_not_recorded: "the @preman_tool decorators were not added, so tool calls are not recorded",
|
|
836
|
+
},
|
|
837
|
+
edited: [],
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
// A file saved while the question was open would lose that edit.
|
|
841
|
+
const changed = plan.files.filter((f) => {
|
|
842
|
+
try {
|
|
843
|
+
return readFileSync(path.join(root, f.path), "utf8") !== f.before;
|
|
844
|
+
} catch {
|
|
845
|
+
return true;
|
|
846
|
+
}
|
|
847
|
+
});
|
|
848
|
+
if (changed.length) {
|
|
849
|
+
throw new TargetError(
|
|
850
|
+
`${changed.map((f) => f.path).join(", ")} changed while PreMan was asking, so nothing was written. ` +
|
|
851
|
+
"Re-run `preman eval target` to see the changes against your current code."
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
for (const f of plan.files) writeFileSync(path.join(root, f.path), f.after, "utf8");
|
|
855
|
+
return { found: { ...found, tools: watched }, edited };
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
async function installShim(found, built, { projectPath, args, log, edited = [] }) {
|
|
731
859
|
const { root, dir, file } = writeShim(projectPath, built.shim);
|
|
732
860
|
const alsoWrote = [];
|
|
733
861
|
if (built.bridge) {
|
|
@@ -762,7 +890,7 @@ async function installShim(found, built, { projectPath, args, log }) {
|
|
|
762
890
|
|
|
763
891
|
const finalSource = readFileSync(file, "utf8");
|
|
764
892
|
const language = String(found.language || "python").toLowerCase() === "node" ? "node" : "python";
|
|
765
|
-
const entry = record(found, file, root, finalSource, { language, alsoWrote });
|
|
893
|
+
const entry = record(found, file, root, finalSource, { language, alsoWrote, edited });
|
|
766
894
|
log(`Approved. ${REPO_CONFIG} remembers it, so this will not be asked again.`);
|
|
767
895
|
|
|
768
896
|
return verifyEntry({ ...entry, root });
|
package/bin/eval_harness.py
CHANGED
|
@@ -570,6 +570,18 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
|
|
|
570
570
|
# LangGraph message trace is replaced only when both count the same calls.
|
|
571
571
|
capture = payload.get("tool_capture") if isinstance(payload, dict) else None
|
|
572
572
|
span_trace = next((t for t in reversed(holder) if t), None)
|
|
573
|
+
if (
|
|
574
|
+
span_trace is None
|
|
575
|
+
and isinstance(capture, dict)
|
|
576
|
+
and capture.get("source") == "decorated"
|
|
577
|
+
and capture.get("state") == "recorded"
|
|
578
|
+
and capture.get("trace") is None
|
|
579
|
+
):
|
|
580
|
+
# Every watched tool records itself, so no span is a confirmed
|
|
581
|
+
# zero, not a turn whose calls carry no trace.
|
|
582
|
+
capture["trace"] = {"links": "full", "known": ["step", "parent", "started", "ended"],
|
|
583
|
+
"unknown": {}, "calls": []}
|
|
584
|
+
return
|
|
573
585
|
if not isinstance(capture, dict) or span_trace is None:
|
|
574
586
|
return
|
|
575
587
|
own = capture.get("trace")
|
package/bin/eval_target.js
CHANGED
|
@@ -207,7 +207,7 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
|
|
|
207
207
|
return searchForEntry({ projectPath, args, log });
|
|
208
208
|
}
|
|
209
209
|
|
|
210
|
-
/** Shapes whose wrapper records tool calls:
|
|
210
|
+
/** Shapes whose wrapper records tool calls: PY_DECORATED, `_graph_calls`, or the
|
|
211
211
|
* recording proxy (PY_PROXY) in eval_templates.js. */
|
|
212
212
|
const CAPTURE_SHAPES = new Set(["py_history", "py_plain", "py_class", "other", "langgraph", "node_bridge", "cli_subprocess"]);
|
|
213
213
|
|
|
@@ -228,14 +228,19 @@ export function verifyEntry(saved) {
|
|
|
228
228
|
if (saved.sha256 && digest !== saved.sha256) return null;
|
|
229
229
|
|
|
230
230
|
// True for a wrapper the CLI generated with tool capture wired in (every
|
|
231
|
-
// shape but `node_bridge`, `cli_subprocess` and `langgraph` defines
|
|
232
|
-
// at module scope -- see
|
|
233
|
-
//
|
|
234
|
-
// `
|
|
235
|
-
//
|
|
236
|
-
//
|
|
231
|
+
// shape but `node_bridge`, `cli_subprocess` and `langgraph` defines
|
|
232
|
+
// `_DECORATED` at module scope -- see PY_DECORATED in eval_templates.js;
|
|
233
|
+
// `_TOOLS` is still recognised for wrappers approved before decorators;
|
|
234
|
+
// `langgraph` reads calls from the graph's messages in `_graph_calls`;
|
|
235
|
+
// `node_bridge` and `cli_subprocess` load the recording proxy in
|
|
236
|
+
// `_tool_proxy`). A wrapper approved before that change has no such line and
|
|
237
|
+
// keeps matching its digest, so this is read from the file rather than the
|
|
238
|
+
// approval record.
|
|
237
239
|
const capturesTools =
|
|
238
|
-
/^
|
|
240
|
+
/^_DECORATED = /m.test(source) ||
|
|
241
|
+
/^_TOOLS = /m.test(source) ||
|
|
242
|
+
/^def _graph_calls\(/m.test(source) ||
|
|
243
|
+
/^def _tool_proxy\(/m.test(source);
|
|
239
244
|
|
|
240
245
|
// Every file the approval produced, not only the wrapper.
|
|
241
246
|
//
|