devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/bench/run.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the lifecycle benchmark: each task asked with and without the process graph (ODSF-308).
|
|
3
|
+
|
|
4
|
+
python3 run.py TASKS.json OUT.jsonl --reps 5 [--only S1,S3] [--seed 7]
|
|
5
|
+
|
|
6
|
+
Arms: `plain` has Read, Grep, Glob and read-only git; `graph` has the same plus the devkb-sdlc
|
|
7
|
+
MCP server over a pinned graph snapshot and the skill, used if the model chooses; `graph_forced` adds an
|
|
8
|
+
instruction to use the graph first, to show what the graph can do when it is actually used. Both get a system-prompt addendum of similar length
|
|
9
|
+
(the skill for graph, a neutral work-efficiently note for plain), so prompt length is not the
|
|
10
|
+
difference. Runs are interleaved in random order with a discarded warm-up per arm.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import random
|
|
18
|
+
import re
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
HERE = Path(__file__).resolve().parent
|
|
25
|
+
CHECKOUT = os.environ.get("BENCH_CHECKOUT", "") # pinned worktree the agents work in
|
|
26
|
+
TOOL_ROOT = os.environ.get("BENCH_TOOL_ROOT", str(HERE.parent.parent)) # checkout holding devkb/mcp_server.py
|
|
27
|
+
SNAPSHOT = os.environ.get("BENCH_GRAPH", "") # pinned graph snapshot directory
|
|
28
|
+
MODEL = os.environ.get("BENCH_MODEL", "sonnet")
|
|
29
|
+
READ_TOOLS = ["Read", "Grep", "Glob", "Bash(git log:*)", "Bash(git show:*)", "Bash(git ls-files:*)", "Bash(git grep:*)",
|
|
30
|
+
"Bash(git diff-tree:*)", "Bash(git diff:*)"]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def skill_body() -> str:
|
|
34
|
+
"""The skill's text without its frontmatter: what a loaded skill adds to the model's context."""
|
|
35
|
+
return (Path(TOOL_ROOT) / "skills" / "devkb-sdlc" / "SKILL.md").read_text().split("---", 2)[2].strip()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def mcp_config(arm: str, path: Path) -> Path:
|
|
39
|
+
cfg = {"mcpServers": {}}
|
|
40
|
+
if arm in ("graph", "graph_forced"):
|
|
41
|
+
cfg["mcpServers"]["devkb-sdlc"] = {
|
|
42
|
+
"command": "uv", "args": ["run", "--python", ">=3.10", "--no-project", "--with", "mcp>=1.23,<2", "python",
|
|
43
|
+
f"{TOOL_ROOT}/devkb/mcp_server.py", "--project", "ODSF"],
|
|
44
|
+
"env": {"DEVKB_DIR": SNAPSHOT}}
|
|
45
|
+
path.write_text(json.dumps(cfg))
|
|
46
|
+
return path
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def run_one(task: dict, arm: str, cfg_dir: Path, tools_extra: list[str] | None = None, cwd: str | None = None, max_turns: int = 30) -> dict:
|
|
50
|
+
cfg = mcp_config(arm, cfg_dir / f"mcp-{arm}.json")
|
|
51
|
+
addendum = (HERE / "control.md").read_text() if arm == "plain" else skill_body()
|
|
52
|
+
tools = READ_TOOLS + (tools_extra or []) + (["mcp__devkb-sdlc"] if arm != "plain" else [])
|
|
53
|
+
prompt = task["prompt"] + (" Use the devkb-sdlc tools first (load them with ToolSearch if they are not yet loaded), "
|
|
54
|
+
"then read only what they name." if arm == "graph_forced" else "")
|
|
55
|
+
cmd = ["claude", "-p", prompt, "--model", MODEL, "--output-format", "stream-json", "--verbose",
|
|
56
|
+
"--max-turns", str(max_turns), "--no-session-persistence", "--mcp-config", str(cfg), "--strict-mcp-config",
|
|
57
|
+
"--append-system-prompt", addendum, "--allowedTools", *tools]
|
|
58
|
+
t0 = time.time()
|
|
59
|
+
proc = subprocess.run(cmd, cwd=cwd or CHECKOUT, capture_output=True, text=True, timeout=900)
|
|
60
|
+
tool_calls = graph_calls = result_bytes = 0
|
|
61
|
+
final: dict = {}
|
|
62
|
+
for line in proc.stdout.splitlines():
|
|
63
|
+
try:
|
|
64
|
+
e = json.loads(line)
|
|
65
|
+
except ValueError:
|
|
66
|
+
continue
|
|
67
|
+
if e.get("type") == "assistant":
|
|
68
|
+
for c in e["message"].get("content", []):
|
|
69
|
+
if c.get("type") == "tool_use":
|
|
70
|
+
tool_calls += 1
|
|
71
|
+
graph_calls += c["name"].startswith("mcp__devkb-sdlc")
|
|
72
|
+
elif e.get("type") == "user" and isinstance(e["message"].get("content"), list):
|
|
73
|
+
for c in e["message"]["content"]:
|
|
74
|
+
if c.get("type") == "tool_result":
|
|
75
|
+
result_bytes += len(json.dumps(c.get("content")))
|
|
76
|
+
elif e.get("type") == "result":
|
|
77
|
+
final = e
|
|
78
|
+
u = final.get("usage", {})
|
|
79
|
+
return {"cost": final.get("total_cost_usd"), "turns": final.get("num_turns"), "secs": round(time.time() - t0, 1),
|
|
80
|
+
"fresh": u.get("input_tokens", 0) + u.get("cache_creation_input_tokens", 0) + u.get("output_tokens", 0),
|
|
81
|
+
"cache_read": u.get("cache_read_input_tokens", 0), "output": u.get("output_tokens", 0),
|
|
82
|
+
"tool_calls": tool_calls, "graph_calls": graph_calls, "result_bytes": result_bytes,
|
|
83
|
+
"is_error": bool(final.get("is_error")) or not final, "answer": final.get("result", "")}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# ── scoring ──────────────────────────────────────────────────────────────────
|
|
87
|
+
def norm(x: str) -> str:
|
|
88
|
+
x = str(x).strip().strip("`").lstrip("./")
|
|
89
|
+
if "::" in x:
|
|
90
|
+
path, name = x.split("::", 1)
|
|
91
|
+
return f"{path}::{name.split('.')[-1].split('(')[0].strip()}"
|
|
92
|
+
return x
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def f1(got, want) -> float:
|
|
96
|
+
g, w = {norm(x) for x in (got or [])}, {norm(x) for x in (want or [])}
|
|
97
|
+
if not g and not w:
|
|
98
|
+
return 1.0
|
|
99
|
+
hit = len(g & w)
|
|
100
|
+
if not hit:
|
|
101
|
+
return 0.0
|
|
102
|
+
p, r = hit / len(g), hit / len(w)
|
|
103
|
+
return 2 * p * r / (p + r)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def parse(text: str) -> dict:
|
|
107
|
+
"""The first JSON object in the answer. The model may add prose before or after it, and
|
|
108
|
+
prose can contain braces, so try every opening brace rather than the span to the last one."""
|
|
109
|
+
decoder = json.JSONDecoder()
|
|
110
|
+
for m in re.finditer(r"\{", text or ""):
|
|
111
|
+
try:
|
|
112
|
+
obj, _ = decoder.raw_decode(text[m.start():])
|
|
113
|
+
except ValueError:
|
|
114
|
+
continue
|
|
115
|
+
if isinstance(obj, dict) and obj:
|
|
116
|
+
return obj
|
|
117
|
+
return {}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def score(task: dict, answer: str) -> dict:
|
|
121
|
+
a, t, kind = parse(answer), task["truth"], task["kind"]
|
|
122
|
+
if kind == "callers":
|
|
123
|
+
parts = {"callers": f1(a.get("callers"), t["callers"]), "test_files": f1(a.get("test_files"), t["test_files"]),
|
|
124
|
+
"owner": f1(a.get("owner"), t["owner"]), "forbidden": f1(a.get("forbidden_for"), t["forbidden_for"])}
|
|
125
|
+
elif kind == "docs_recall":
|
|
126
|
+
got = [norm(x) for x in (a.get("docs") or [])][:8]
|
|
127
|
+
parts = {"recall": (len({*got} & {norm(x) for x in t["docs"]}) / len(t["docs"])) if t["docs"] else 1.0,
|
|
128
|
+
"exist": (sum(1 for g in got if (Path(CHECKOUT) / g).exists()) / len(got)) if got else 0.0}
|
|
129
|
+
elif kind == "docs_set":
|
|
130
|
+
parts = {"f1": f1(a.get("adrs"), t["adrs"])}
|
|
131
|
+
elif kind == "set":
|
|
132
|
+
parts = {"rules": 1.0 if sorted(a.get("failing_rules") or []) == sorted(t["failing_rules"]) else 0.0}
|
|
133
|
+
elif kind == "review":
|
|
134
|
+
parts = {"changed": f1(a.get("changed"), t["changed"]), "callers": f1(a.get("callers_outside"), t["callers_outside"]),
|
|
135
|
+
"tests": f1(a.get("test_files"), t["test_files"])}
|
|
136
|
+
else:
|
|
137
|
+
parts = {k: 1.0 if str(a.get(k)) == str(v) else 0.0 for k, v in t.items()}
|
|
138
|
+
main = parts.get("recall", sum(parts.values()) / len(parts))
|
|
139
|
+
return {"score": round(main, 3), "parts": {k: round(v, 3) for k, v in parts.items()}}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def main() -> int:
|
|
143
|
+
ap = argparse.ArgumentParser()
|
|
144
|
+
ap.add_argument("tasks")
|
|
145
|
+
ap.add_argument("out")
|
|
146
|
+
ap.add_argument("--reps", type=int, default=5)
|
|
147
|
+
ap.add_argument("--only", default="")
|
|
148
|
+
ap.add_argument("--seed", type=int, default=7)
|
|
149
|
+
ns = ap.parse_args()
|
|
150
|
+
if not (CHECKOUT and SNAPSHOT):
|
|
151
|
+
ap.error("set BENCH_CHECKOUT (pinned worktree) and BENCH_GRAPH (graph snapshot directory)")
|
|
152
|
+
tasks = [t for t in json.load(open(ns.tasks)) if not ns.only or t["stage"] in ns.only.split(",")]
|
|
153
|
+
rnd = random.Random(ns.seed)
|
|
154
|
+
cfg_dir = Path(ns.out).resolve().parent
|
|
155
|
+
pairs = [(t, r) for t in tasks for r in range(ns.reps)]
|
|
156
|
+
rnd.shuffle(pairs)
|
|
157
|
+
with open(ns.out, "a") as out:
|
|
158
|
+
for arm in ("plain", "graph", "graph_forced"): # warm-up: one discarded run per arm so neither pays the cold cache alone
|
|
159
|
+
run_one(tasks[0], arm, cfg_dir)
|
|
160
|
+
for t, rep in pairs:
|
|
161
|
+
for arm in rnd.sample(["plain", "graph", "graph_forced"], 3):
|
|
162
|
+
res = run_one(t, arm, cfg_dir)
|
|
163
|
+
valid = not res["is_error"] and bool(parse(res["answer"]))
|
|
164
|
+
row = {"task": t["id"], "stage": t["stage"], "arm": arm, "rep": rep, "valid": valid,
|
|
165
|
+
**{k: v for k, v in res.items() if k != "answer"},
|
|
166
|
+
**(score(t, res["answer"]) if valid else {"score": 0.0, "parts": {}}), "answer": res["answer"][:1500]}
|
|
167
|
+
out.write(json.dumps(row) + "\n")
|
|
168
|
+
out.flush()
|
|
169
|
+
print(t["id"], arm, rep, "valid" if valid else "INVALID", row["score"], row["cost"], row["turns"], row["graph_calls"], flush=True)
|
|
170
|
+
return 0
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__":
|
|
174
|
+
sys.exit(main())
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""An independent check of each refactor task's truth: the files with a direct caller, found by a second implementation
|
|
2
|
+
that shares no code with the graph (`devkb/build_symbols.py`) or the task generators' truth (`module_truth.py`).
|
|
3
|
+
|
|
4
|
+
The pack check compares a pack with the truth; when both are built with the same import rules, a blind spot they share
|
|
5
|
+
scores 1.0 (lnbits `get_wallet`, 2026-11-15: both missed every caller that imported the function through its package's
|
|
6
|
+
`__init__.py`). This check resolves each import to the file that defines the name, following re-exports, and counts a
|
|
7
|
+
call wherever it is made, inside a function or at import time. A file it finds that the truth lacks is reported as
|
|
8
|
+
`missing`: it is proven by an import chain. A file the truth has that it does not find is `unconfirmed`, weaker
|
|
9
|
+
evidence, since this check does not follow calls through objects.
|
|
10
|
+
|
|
11
|
+
python3 devkb/bench/truth_check.py CHECKOUT TASKS.json
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import ast
|
|
16
|
+
import collections
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
PROMPT = re.compile(r"`([^`]+)` \(defined in ([^)]+)\)")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _Repo:
|
|
27
|
+
def __init__(self, root: Path) -> None:
|
|
28
|
+
listed = subprocess.run(["git", "ls-files", "--", "*.py"], cwd=root, capture_output=True, text=True).stdout.split()
|
|
29
|
+
self.trees: dict[str, ast.Module] = {}
|
|
30
|
+
for rel in listed:
|
|
31
|
+
try:
|
|
32
|
+
self.trees[rel] = ast.parse((root / rel).read_text(errors="replace"))
|
|
33
|
+
except (SyntaxError, ValueError, OSError, RecursionError):
|
|
34
|
+
continue
|
|
35
|
+
# dotted module -> files whose path ends that way, so `src/` layouts and nested services resolve
|
|
36
|
+
self.by_suffix: dict[str, list[str]] = collections.defaultdict(list)
|
|
37
|
+
for rel in self.trees:
|
|
38
|
+
parts = rel[:-3].split("/")
|
|
39
|
+
if parts[-1] == "__init__":
|
|
40
|
+
parts = parts[:-1]
|
|
41
|
+
for i in range(len(parts)):
|
|
42
|
+
self.by_suffix[".".join(parts[i:])].append(rel)
|
|
43
|
+
self.defined = {rel: {n.name for n in t.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))}
|
|
44
|
+
for rel, t in self.trees.items()}
|
|
45
|
+
self.imports = {rel: self._imports(rel, t) for rel, t in self.trees.items()}
|
|
46
|
+
|
|
47
|
+
def module(self, dotted: str) -> str | None:
|
|
48
|
+
hits = self.by_suffix.get(dotted, [])
|
|
49
|
+
return hits[0] if len(hits) == 1 else None
|
|
50
|
+
|
|
51
|
+
@staticmethod
|
|
52
|
+
def _imports(rel: str, tree: ast.Module) -> tuple[dict[str, tuple[str, str]], dict[str, str]]:
|
|
53
|
+
"""({local: (module, name)} from `from m import name`, {local: module} from `import m` / `import m as x`)."""
|
|
54
|
+
names, modules = {}, {}
|
|
55
|
+
pkg = rel[:-3].split("/")[:-1]
|
|
56
|
+
for n in ast.walk(tree):
|
|
57
|
+
if isinstance(n, ast.ImportFrom):
|
|
58
|
+
base = (".".join(pkg[: len(pkg) - n.level + 1] + ([n.module] if n.module else [])) if n.level
|
|
59
|
+
else n.module or "")
|
|
60
|
+
for a in n.names:
|
|
61
|
+
if a.name != "*":
|
|
62
|
+
names[a.asname or a.name] = (base, a.name)
|
|
63
|
+
elif isinstance(n, ast.Import):
|
|
64
|
+
for a in n.names:
|
|
65
|
+
modules[a.asname or a.name] = a.name
|
|
66
|
+
return names, modules
|
|
67
|
+
|
|
68
|
+
def origin(self, dotted: str, name: str, depth: int = 0) -> tuple[str, str] | None:
|
|
69
|
+
"""The (file, name) that `name` looked up in module `dotted` is: defined there, or re-exported from elsewhere."""
|
|
70
|
+
rel = self.module(dotted)
|
|
71
|
+
if rel is None: # the package path may be ambiguous while the submodule it names is not
|
|
72
|
+
return ("<module>", f"{dotted}.{name}") if self.module(f"{dotted}.{name}") else None
|
|
73
|
+
if name in self.defined[rel]:
|
|
74
|
+
return rel, name
|
|
75
|
+
via = self.imports[rel][0].get(name)
|
|
76
|
+
if via and depth < 6:
|
|
77
|
+
return self.origin(via[0], via[1], depth + 1)
|
|
78
|
+
sub = f"{dotted}.{name}" # `from pkg import mod`: a module, not a name
|
|
79
|
+
return ("<module>", sub) if self.module(sub) else None
|
|
80
|
+
|
|
81
|
+
def caller_files(self, path: str, name: str) -> set[str]:
|
|
82
|
+
want = (path, name)
|
|
83
|
+
out: set[str] = set()
|
|
84
|
+
for rel, tree in self.trees.items():
|
|
85
|
+
if rel == path: # the defining file calls it by its bare name, from another function
|
|
86
|
+
if any(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name != name and any(
|
|
87
|
+
isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == name for c in ast.walk(n))
|
|
88
|
+
for n in ast.walk(tree)):
|
|
89
|
+
out.add(rel)
|
|
90
|
+
continue
|
|
91
|
+
names, modules = self.imports[rel]
|
|
92
|
+
local = {k for k, (m, n) in names.items() if self.origin(m, n) == want}
|
|
93
|
+
mods: set[str] = set() # local names bound to a module whose `name` is the target
|
|
94
|
+
for k, (m, n) in names.items():
|
|
95
|
+
o = self.origin(m, n)
|
|
96
|
+
if o and o[0] == "<module>" and self.origin(o[1], name) == want:
|
|
97
|
+
mods.add(k)
|
|
98
|
+
for k, m in modules.items():
|
|
99
|
+
if self.origin(m, name) == want:
|
|
100
|
+
mods.add(k)
|
|
101
|
+
for n in ast.walk(tree):
|
|
102
|
+
if not isinstance(n, ast.Call):
|
|
103
|
+
continue
|
|
104
|
+
f = n.func
|
|
105
|
+
if isinstance(f, ast.Name) and f.id in local:
|
|
106
|
+
out.add(rel)
|
|
107
|
+
elif isinstance(f, ast.Attribute) and f.attr == name:
|
|
108
|
+
v, chain = f.value, []
|
|
109
|
+
while isinstance(v, ast.Attribute):
|
|
110
|
+
chain.insert(0, v.attr)
|
|
111
|
+
v = v.value
|
|
112
|
+
if not isinstance(v, ast.Name):
|
|
113
|
+
continue
|
|
114
|
+
if ".".join([v.id, *chain]) in mods:
|
|
115
|
+
out.add(rel)
|
|
116
|
+
continue
|
|
117
|
+
# `models.flow_runs.read_flow_run()`: the root is a module, the rest a path below it
|
|
118
|
+
root = names.get(v.id)
|
|
119
|
+
base = (self.origin(*root) or (None, None)) if root else (None, None)
|
|
120
|
+
dotted = base[1] if base[0] == "<module>" else modules.get(v.id)
|
|
121
|
+
if dotted and chain and self.origin(".".join([dotted, *chain]), name) == want:
|
|
122
|
+
out.add(rel)
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def check(root: Path, tasks: list[dict]) -> list[dict]:
|
|
127
|
+
repo = _Repo(root)
|
|
128
|
+
rows = []
|
|
129
|
+
for t in tasks:
|
|
130
|
+
m = PROMPT.search(t.get("prompt", ""))
|
|
131
|
+
if t.get("kind") != "refactor" or not m or "." in m.group(1):
|
|
132
|
+
continue
|
|
133
|
+
found = repo.caller_files(m.group(2), m.group(1))
|
|
134
|
+
truth = set(t["truth"]["caller_files"])
|
|
135
|
+
rows.append({"task": t["id"], "truth": len(truth), "found": len(found),
|
|
136
|
+
"missing": sorted(found - truth), "unconfirmed": sorted(truth - found)})
|
|
137
|
+
return rows
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def main() -> int:
|
|
141
|
+
root, tasks = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text())
|
|
142
|
+
rows = check(root, tasks)
|
|
143
|
+
print(json.dumps(rows, indent=1))
|
|
144
|
+
return 1 if any(r["missing"] for r in rows) else 0
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Summarise workflow benchmark rows: per task kind and arm, median cost, turns, tool-result bytes, mean score, and the
|
|
3
|
+
cost saved against plain with a bootstrap 95% interval (resampling runs within each task).
|
|
4
|
+
|
|
5
|
+
python3 workflow_report.py results.jsonl
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import collections
|
|
10
|
+
import json
|
|
11
|
+
import random
|
|
12
|
+
import statistics
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def boot(rows_by_task: dict, arm: str, n: int = 4000) -> tuple[float, float, float]:
|
|
17
|
+
"""Cost saved by `arm` against plain: 1 - sum(mean arm cost per task) / sum(mean plain cost per task)."""
|
|
18
|
+
rnd = random.Random(1)
|
|
19
|
+
|
|
20
|
+
def saved(sample):
|
|
21
|
+
a = sum(statistics.mean(s[arm]) for s in sample.values())
|
|
22
|
+
p = sum(statistics.mean(s["plain"]) for s in sample.values())
|
|
23
|
+
return 1 - a / p
|
|
24
|
+
|
|
25
|
+
point = saved(rows_by_task)
|
|
26
|
+
draws = []
|
|
27
|
+
for _ in range(n):
|
|
28
|
+
draws.append(saved({t: {k: [rnd.choice(v) for _ in v] for k, v in s.items()} for t, s in rows_by_task.items()}))
|
|
29
|
+
draws.sort()
|
|
30
|
+
return point, draws[int(n * 0.025)], draws[int(n * 0.975)]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def main() -> int:
|
|
34
|
+
rows = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
|
|
35
|
+
print(f"{len(rows)} runs, {sum(r['valid'] for r in rows)} valid, total ${sum(r['cost'] or 0 for r in rows):.2f}\n")
|
|
36
|
+
kinds = sorted({r["kind"] for r in rows}) + ["ALL"]
|
|
37
|
+
print(f"{'kind':9} {'arm':9} {'n':>2} {'cost$':>7} {'turns':>5} {'bytes':>7} {'score':>5} {'graph%':>6} saved vs plain [95%] tools used")
|
|
38
|
+
for kind in kinds:
|
|
39
|
+
sel = [r for r in rows if kind == "ALL" or r["kind"] == kind]
|
|
40
|
+
by_task: dict = collections.defaultdict(lambda: collections.defaultdict(list))
|
|
41
|
+
for r in sel:
|
|
42
|
+
if r["cost"] is not None:
|
|
43
|
+
by_task[r["task"]][r["arm"]].append(r["cost"])
|
|
44
|
+
arms = [a for a in ("plain", "core", "workflow", "lean", "pack") if any(r["arm"] == a for r in sel)]
|
|
45
|
+
by_task = {t: s for t, s in by_task.items() if all(s.get(a) for a in arms)}
|
|
46
|
+
for arm in arms:
|
|
47
|
+
a = [r for r in sel if r["arm"] == arm]
|
|
48
|
+
if not a:
|
|
49
|
+
continue
|
|
50
|
+
med = lambda k: statistics.median([r[k] or 0 for r in a]) # noqa: E731
|
|
51
|
+
used = collections.Counter(t for r in a for t in r.get("graph_tools", []))
|
|
52
|
+
saved = ""
|
|
53
|
+
if arm != "plain" and by_task:
|
|
54
|
+
p, lo, hi = boot(by_task, arm)
|
|
55
|
+
saved = f"{p:+.0%} [{lo:+.0%}, {hi:+.0%}]"
|
|
56
|
+
print(f"{kind:9} {arm:9} {len(a):2} {med('cost'):7.3f} {med('turns'):5.0f} {med('result_bytes'):7.0f} "
|
|
57
|
+
f"{statistics.mean(r['score'] for r in a):5.2f} {sum(r['graph_calls'] > 0 for r in a) / len(a):6.0%} {saved:22} "
|
|
58
|
+
f"{', '.join(f'{k}×{v}' for k, v in used.most_common(4))}")
|
|
59
|
+
print()
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
if __name__ == "__main__":
|
|
64
|
+
sys.exit(main())
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the workflow benchmark (ODSF-310): whole developer tasks, three arms, interleaved.
|
|
3
|
+
|
|
4
|
+
plain Read, Grep, Glob, read-only git, and a neutral work-efficiently note (control.md)
|
|
5
|
+
core the same plus the devkb-sdlc MCP server with the 11 single-question tools and the devkb-sdlc skill
|
|
6
|
+
workflow the same plus the workflow toolset (one tool per developer task) and the task's own skill
|
|
7
|
+
|
|
8
|
+
Both graph arms are told to use the graph first: ODSF-308 showed that whether the model picks the graph up on its
|
|
9
|
+
own is a separate question (adoption), and this one asks what each tool surface costs when it is used.
|
|
10
|
+
|
|
11
|
+
BENCH_CHECKOUT=... BENCH_GRAPH=... python3 devkb/bench/workflow_run.py TASKS.json OUT.jsonl --reps 4
|
|
12
|
+
|
|
13
|
+
Report: python3 devkb/bench/workflow_report.py OUT.jsonl. Write-up: docs/workflow-benchmark-odsf310.md.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import random
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import threading
|
|
24
|
+
import time
|
|
25
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
HERE = Path(__file__).resolve().parent
|
|
29
|
+
TOOL_ROOT = Path(os.environ.get("BENCH_TOOL_ROOT", str(HERE.parent.parent))) # checkout holding devkb/mcp_server.py
|
|
30
|
+
# Absolute: the MCP server starts inside the checkout, so a relative graph path would point nowhere and the tool arms
|
|
31
|
+
# would run with no tools while still told to use them (the 2026-11-15 addendum's first attempt).
|
|
32
|
+
CHECKOUT = str(Path(os.environ["BENCH_CHECKOUT"]).resolve()) if os.environ.get("BENCH_CHECKOUT") else "" # pinned worktree
|
|
33
|
+
SNAPSHOT = str(Path(os.environ["BENCH_GRAPH"]).resolve()) if os.environ.get("BENCH_GRAPH") else "" # graph at its HEAD
|
|
34
|
+
MODEL = os.environ.get("BENCH_MODEL", "sonnet")
|
|
35
|
+
PROJECT = os.environ.get("BENCH_PROJECT", "ODSF")
|
|
36
|
+
sys.path.insert(0, str(HERE))
|
|
37
|
+
import run as base # noqa: E402 parse, norm, f1 and READ_TOOLS are shared with the lifecycle benchmark
|
|
38
|
+
|
|
39
|
+
ARMS = tuple(os.environ.get("BENCH_ARMS", "plain,core,workflow").split(",")) # ODSF-313 adds lean and pack
|
|
40
|
+
FORCE = " Use the devkb-sdlc tools first (load them with ToolSearch if they are not yet loaded), then read only what they name."
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def skill(name: str) -> str:
|
|
44
|
+
return (TOOL_ROOT / "skills" / name / "SKILL.md").read_text().split("---", 2)[2].strip()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
LIMIT_TEXT = ("hit your", "usage limit", "rate limit")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def run_one(task: dict, arm: str, cfg_dir: Path) -> dict:
|
|
51
|
+
"""One run. A run the account's usage limit stopped is not a result: wait and run it again (deviation D3). A task
|
|
52
|
+
the agent failed is never retried."""
|
|
53
|
+
for attempt in range(48):
|
|
54
|
+
res = _run_once(task, arm, cfg_dir)
|
|
55
|
+
text = (res.get("answer") or "").lower()
|
|
56
|
+
if not (res.get("is_error") and any(s in text for s in LIMIT_TEXT)):
|
|
57
|
+
res["limit_retries"] = attempt
|
|
58
|
+
return res
|
|
59
|
+
print(f"usage limit on {task['id']} {arm}; waiting 10 minutes (attempt {attempt + 1})", flush=True)
|
|
60
|
+
time.sleep(600)
|
|
61
|
+
return res
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _run_once(task: dict, arm: str, cfg_dir: Path) -> dict:
|
|
65
|
+
cfg = {"mcpServers": {}}
|
|
66
|
+
if arm not in ("plain", "pack"): # a pack arm loads no graph tools: the graph's answer is already in the prompt
|
|
67
|
+
cfg["mcpServers"]["devkb-sdlc"] = {
|
|
68
|
+
# --no-config: uv starts inside the checkout, and a repository's own uv settings (Prefect pins
|
|
69
|
+
# `required-version`) must not stop the graph server; on 2026-11-15 they silently left Prefect's tool arms
|
|
70
|
+
# without tools in all 90 runs.
|
|
71
|
+
"command": "uv", "args": ["run", "--no-config", "--python", ">=3.10", "--no-project", "--with", "mcp>=1.23,<2", "python",
|
|
72
|
+
str(TOOL_ROOT / "devkb" / "mcp_server.py"), "--project", PROJECT],
|
|
73
|
+
"env": {"DEVKB_DIR": SNAPSHOT, "DEVKB_REPO": CHECKOUT, "DEVKB_TOOLSET": arm,
|
|
74
|
+
**({"DEVKB_EXTERNAL": "1"} if os.environ.get("DEVKB_EXTERNAL") == "1" else {}),
|
|
75
|
+
**({"DEVKB_NO_COVERAGE": "1"} if os.environ.get("DEVKB_NO_COVERAGE") == "1" else {})}}
|
|
76
|
+
path = cfg_dir / f"mcp-{arm}-{threading.get_ident()}.json"
|
|
77
|
+
path.write_text(json.dumps(cfg))
|
|
78
|
+
control = (TOOL_ROOT / "devkb" / "bench" / "control.md").read_text()
|
|
79
|
+
addendum = {"plain": control, "pack": control, "core": skill("devkb-sdlc"), "lean": skill("devkb-lean"),
|
|
80
|
+
"lean-pack": skill("devkb-lean-pack"),
|
|
81
|
+
"workflow": skill(task["skill"])}[arm]
|
|
82
|
+
tools = base.READ_TOOLS + (["mcp__devkb-sdlc"] if arm not in ("plain", "pack") else [])
|
|
83
|
+
prompt = task["prompt"] + (FORCE if arm not in ("plain", "pack") else "")
|
|
84
|
+
if arm == "pack":
|
|
85
|
+
prompt += "\n\n" + task["pack"]
|
|
86
|
+
cmd = ["claude", "-p", prompt, "--model", MODEL, "--output-format", "stream-json",
|
|
87
|
+
"--verbose", "--max-turns", os.environ.get("BENCH_MAX_TURNS", "30"), "--no-session-persistence", "--mcp-config", str(path), "--strict-mcp-config",
|
|
88
|
+
"--append-system-prompt", addendum, "--allowedTools", *tools]
|
|
89
|
+
t0 = time.time()
|
|
90
|
+
try:
|
|
91
|
+
proc = subprocess.run(cmd, cwd=CHECKOUT, capture_output=True, text=True, timeout=900)
|
|
92
|
+
stdout = proc.stdout
|
|
93
|
+
except subprocess.TimeoutExpired as exc:
|
|
94
|
+
stdout = exc.stdout.decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "")
|
|
95
|
+
calls, graph, rbytes, used = 0, 0, 0, []
|
|
96
|
+
final: dict = {}
|
|
97
|
+
mcp_ok = arm in ("plain", "pack") # a tool arm counts only when its graph server connected
|
|
98
|
+
contexts: dict[str, int] = {} # message id -> tokens of context that message was given (insertion-ordered)
|
|
99
|
+
for line in stdout.splitlines():
|
|
100
|
+
try:
|
|
101
|
+
e = json.loads(line)
|
|
102
|
+
except ValueError:
|
|
103
|
+
continue
|
|
104
|
+
if e.get("type") == "system" and e.get("subtype") == "init":
|
|
105
|
+
mcp_ok = mcp_ok or any(m.get("name") == "devkb-sdlc" and m.get("status") == "connected"
|
|
106
|
+
for m in e.get("mcp_servers") or [])
|
|
107
|
+
elif e.get("type") == "assistant":
|
|
108
|
+
mu = e["message"].get("usage") or {}
|
|
109
|
+
if e["message"].get("id") and mu:
|
|
110
|
+
contexts[e["message"]["id"]] = (mu.get("input_tokens", 0) + mu.get("cache_creation_input_tokens", 0)
|
|
111
|
+
+ mu.get("cache_read_input_tokens", 0))
|
|
112
|
+
for c in e["message"].get("content", []):
|
|
113
|
+
if c.get("type") == "tool_use":
|
|
114
|
+
calls += 1
|
|
115
|
+
if c["name"].startswith("mcp__devkb-sdlc"):
|
|
116
|
+
graph += 1
|
|
117
|
+
used.append(c["name"].split("__")[-1])
|
|
118
|
+
elif e.get("type") == "user" and isinstance(e["message"].get("content"), list):
|
|
119
|
+
rbytes += sum(len(json.dumps(c.get("content"))) for c in e["message"]["content"] if c.get("type") == "tool_result")
|
|
120
|
+
elif e.get("type") == "result":
|
|
121
|
+
final = e
|
|
122
|
+
u = final.get("usage", {})
|
|
123
|
+
cw = u.get("cache_creation") or {}
|
|
124
|
+
ctx = list(contexts.values())
|
|
125
|
+
return {"cost": final.get("total_cost_usd"), "turns": final.get("num_turns"), "secs": round(time.time() - t0, 1),
|
|
126
|
+
"fresh": u.get("input_tokens", 0) + u.get("cache_creation_input_tokens", 0) + u.get("output_tokens", 0),
|
|
127
|
+
"cache_read": u.get("cache_read_input_tokens", 0), "output": u.get("output_tokens", 0),
|
|
128
|
+
# token kinds: uncached input, cache writes (by lifetime), thinking; context size per model call
|
|
129
|
+
"input": u.get("input_tokens", 0), "cache_write": u.get("cache_creation_input_tokens", 0),
|
|
130
|
+
"cache_write_5m": cw.get("ephemeral_5m_input_tokens", 0), "cache_write_1h": cw.get("ephemeral_1h_input_tokens", 0),
|
|
131
|
+
"thinking": (u.get("output_tokens_details") or {}).get("thinking_tokens", 0), "model_calls": len(ctx),
|
|
132
|
+
"ctx_initial": ctx[0] if ctx else 0, "ctx_peak": max(ctx) if ctx else 0, "ctx_final": ctx[-1] if ctx else 0,
|
|
133
|
+
"tool_calls": calls,
|
|
134
|
+
"graph_calls": graph, "graph_tools": used, "result_bytes": rbytes,
|
|
135
|
+
"mcp_connected": mcp_ok,
|
|
136
|
+
"is_error": bool(final.get("is_error")) or not final or not mcp_ok, "answer": final.get("result", "")}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def score(task: dict, answer: str) -> dict:
|
|
140
|
+
a, t, f1, norm = base.parse(answer), task["truth"], base.f1, base.norm
|
|
141
|
+
if task["kind"] == "rca":
|
|
142
|
+
parts = {"raiser": 1.0 if norm(a.get("raiser", "")) == norm(t["raiser"]) else 0.0,
|
|
143
|
+
"callers": f1(a.get("callers"), t["callers"]), "tests": f1(a.get("test_files"), t["test_files"])}
|
|
144
|
+
elif task["kind"] == "err_to_handler":
|
|
145
|
+
parts = {"raiser": 1.0 if norm(a.get("raiser", "")) == norm(t["raiser"]) else 0.0,
|
|
146
|
+
"handlers": f1(a.get("handlers"), t["handlers"]), "tests": f1(a.get("test_files"), t["test_files"])}
|
|
147
|
+
elif task["kind"] == "impact":
|
|
148
|
+
parts = {"handlers": f1(a.get("handlers"), t["handlers"]), "tests": f1(a.get("test_files"), t["test_files"])}
|
|
149
|
+
elif task["kind"] == "ticket_code": # ADDENDUM-1 S2
|
|
150
|
+
parts = {"functions": f1(a.get("functions"), t["functions"]), "tests": f1(a.get("test_files"), t["test_files"])}
|
|
151
|
+
elif task["kind"] == "commit_trace":
|
|
152
|
+
parts = {k: 1.0 if str(a.get(k, "")).strip().upper() == str(t[k]).upper() else 0.0 for k in ("ticket", "epic", "spec")}
|
|
153
|
+
elif task["kind"] == "refactor":
|
|
154
|
+
parts = {"files": f1(a.get("caller_files"), t["caller_files"]), "tests": f1(a.get("test_files"), t["test_files"])}
|
|
155
|
+
if "blocked_files" in t: # ODSF-315: a repository with no role profiles asks no boundary question
|
|
156
|
+
parts["blocked"] = f1(a.get("blocked_files"), t["blocked_files"])
|
|
157
|
+
else: # onboard: how many of the three named are in the true top five (ties at fifth place kept)
|
|
158
|
+
got = [norm(x) for x in (a.get("top") or [])][:3]
|
|
159
|
+
parts = {"p_at_3": sum(1 for g in got if g in {norm(x) for x in t["top5"]}) / 3}
|
|
160
|
+
return {"score": round(sum(parts.values()) / len(parts), 3), "parts": {k: round(v, 3) for k, v in parts.items()}}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def main() -> int:
|
|
164
|
+
ap = argparse.ArgumentParser()
|
|
165
|
+
ap.add_argument("tasks")
|
|
166
|
+
ap.add_argument("out")
|
|
167
|
+
ap.add_argument("--reps", type=int, default=4)
|
|
168
|
+
ap.add_argument("--workers", type=int, default=3)
|
|
169
|
+
ap.add_argument("--seed", type=int, default=310)
|
|
170
|
+
ns = ap.parse_args()
|
|
171
|
+
if not (CHECKOUT and SNAPSHOT):
|
|
172
|
+
ap.error("set BENCH_CHECKOUT (pinned worktree) and BENCH_GRAPH (graph snapshot directory)")
|
|
173
|
+
head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=CHECKOUT, capture_output=True, text=True).stdout.strip()
|
|
174
|
+
built = json.loads((Path(SNAPSHOT) / "manifest.json").read_text()).get("git_head", "")
|
|
175
|
+
if head != built:
|
|
176
|
+
sys.exit(f"graph snapshot {built[:8]} is not the checkout {head[:8]}: the comparison would not be fair")
|
|
177
|
+
tasks = json.load(open(ns.tasks))
|
|
178
|
+
rnd = random.Random(ns.seed)
|
|
179
|
+
cfg_dir = Path(ns.out).resolve().parent
|
|
180
|
+
jobs = [(t, rep, arm) for t in tasks for rep in range(ns.reps) for arm in ARMS]
|
|
181
|
+
rnd.shuffle(jobs)
|
|
182
|
+
for arm in ARMS: # one discarded warm-up per arm, so no arm alone pays the cold prompt cache
|
|
183
|
+
run_one(tasks[0], arm, cfg_dir)
|
|
184
|
+
lock = threading.Lock()
|
|
185
|
+
|
|
186
|
+
def job(j):
|
|
187
|
+
t, rep, arm = j
|
|
188
|
+
res = run_one(t, arm, cfg_dir)
|
|
189
|
+
valid = not res["is_error"] and bool(base.parse(res["answer"]))
|
|
190
|
+
row = {"task": t["id"], "kind": t["kind"], "arm": arm, "rep": rep, "valid": valid, "model": MODEL, "project": PROJECT,
|
|
191
|
+
**{k: v for k, v in res.items() if k != "answer"},
|
|
192
|
+
**(score(t, res["answer"]) if valid else {"score": 0.0, "parts": {}}),
|
|
193
|
+
# the whole answer (capped far above any real one), so a run can be rescored when a truth is corrected;
|
|
194
|
+
# 1,500 characters cut most long file lists in two (lnbits get_wallet, 2026-11-15)
|
|
195
|
+
"answer": res["answer"][:50000]}
|
|
196
|
+
with lock, open(ns.out, "a") as out:
|
|
197
|
+
out.write(json.dumps(row) + "\n")
|
|
198
|
+
print(t["id"], arm, rep, "valid" if valid else "INVALID", row["score"], row["cost"], row["turns"], row["graph_calls"], flush=True)
|
|
199
|
+
|
|
200
|
+
with ThreadPoolExecutor(ns.workers) as pool:
|
|
201
|
+
list(pool.map(job, jobs))
|
|
202
|
+
return 0
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
if __name__ == "__main__":
|
|
206
|
+
sys.exit(main())
|