devkb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. devkb/__init__.py +11 -0
  2. devkb/audit.py +173 -0
  3. devkb/bench/add_packs.py +52 -0
  4. devkb/bench/cli_tasks.py +105 -0
  5. devkb/bench/confirm_tasks.py +149 -0
  6. devkb/bench/module_truth.py +249 -0
  7. devkb/bench/oco_tasks.py +216 -0
  8. devkb/bench/pipeline.py +313 -0
  9. devkb/bench/process_tasks.py +135 -0
  10. devkb/bench/report.py +74 -0
  11. devkb/bench/run.py +174 -0
  12. devkb/bench/truth_check.py +148 -0
  13. devkb/bench/workflow_report.py +64 -0
  14. devkb/bench/workflow_run.py +206 -0
  15. devkb/bench/workflow_tasks.py +227 -0
  16. devkb/build.py +258 -0
  17. devkb/build_symbols.py +1394 -0
  18. devkb/call_recall.py +159 -0
  19. devkb/cli.py +53 -0
  20. devkb/coverage.py +155 -0
  21. devkb/edge_truth.py +272 -0
  22. devkb/extractors/backlog.py +113 -0
  23. devkb/extractors/component_subsystem.py +116 -0
  24. devkb/extractors/configuration.py +267 -0
  25. devkb/extractors/files_tests.py +220 -0
  26. devkb/extractors/flows.py +117 -0
  27. devkb/extractors/lifecycle.py +258 -0
  28. devkb/extractors/members.py +160 -0
  29. devkb/extractors/pull_requests.py +271 -0
  30. devkb/extractors/releases.py +49 -0
  31. devkb/extractors/source_baseline.py +66 -0
  32. devkb/extractors/spec_docs.py +220 -0
  33. devkb/extractors/symbol_tests.py +131 -0
  34. devkb/extractors/test_inventory.py +38 -0
  35. devkb/extractors/ticket_similarity.py +193 -0
  36. devkb/extractors/tickets_git.py +151 -0
  37. devkb/extractors/warnings.py +230 -0
  38. devkb/extractors/work_units.py +90 -0
  39. devkb/github.py +515 -0
  40. devkb/gitutil.py +16 -0
  41. devkb/index.py +51 -0
  42. devkb/init.py +178 -0
  43. devkb/mcp_server.py +216 -0
  44. devkb/model.py +221 -0
  45. devkb/module_file_bridge.py +57 -0
  46. devkb/ontology.md +178 -0
  47. devkb/packs.py +462 -0
  48. devkb/process_api.py +1601 -0
  49. devkb/project_context.py +191 -0
  50. devkb/query.py +251 -0
  51. devkb/service.py +324 -0
  52. devkb/startup.py +103 -0
  53. devkb/test_discovery.py +76 -0
  54. devkb/ts_symbols.py +291 -0
  55. devkb-0.2.0.dist-info/METADATA +162 -0
  56. devkb-0.2.0.dist-info/RECORD +61 -0
  57. devkb-0.2.0.dist-info/WHEEL +5 -0
  58. devkb-0.2.0.dist-info/entry_points.txt +2 -0
  59. devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
  60. devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
  61. devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/bench/run.py ADDED
@@ -0,0 +1,174 @@
1
+ #!/usr/bin/env python3
2
+ """Run the lifecycle benchmark: each task asked with and without the process graph (ODSF-308).
3
+
4
+ python3 run.py TASKS.json OUT.jsonl --reps 5 [--only S1,S3] [--seed 7]
5
+
6
+ Arms: `plain` has Read, Grep, Glob and read-only git; `graph` has the same plus the devkb-sdlc
7
+ MCP server over a pinned graph snapshot and the skill, used if the model chooses; `graph_forced` adds an
8
+ instruction to use the graph first, to show what the graph can do when it is actually used. Both get a system-prompt addendum of similar length
9
+ (the skill for graph, a neutral work-efficiently note for plain), so prompt length is not the
10
+ difference. Runs are interleaved in random order with a discarded warm-up per arm.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import json
16
+ import os
17
+ import random
18
+ import re
19
+ import subprocess
20
+ import sys
21
+ import time
22
+ from pathlib import Path
23
+
24
+ HERE = Path(__file__).resolve().parent
25
+ CHECKOUT = os.environ.get("BENCH_CHECKOUT", "") # pinned worktree the agents work in
26
+ TOOL_ROOT = os.environ.get("BENCH_TOOL_ROOT", str(HERE.parent.parent)) # checkout holding devkb/mcp_server.py
27
+ SNAPSHOT = os.environ.get("BENCH_GRAPH", "") # pinned graph snapshot directory
28
+ MODEL = os.environ.get("BENCH_MODEL", "sonnet")
29
+ READ_TOOLS = ["Read", "Grep", "Glob", "Bash(git log:*)", "Bash(git show:*)", "Bash(git ls-files:*)", "Bash(git grep:*)",
30
+ "Bash(git diff-tree:*)", "Bash(git diff:*)"]
31
+
32
+
33
+ def skill_body() -> str:
34
+ """The skill's text without its frontmatter: what a loaded skill adds to the model's context."""
35
+ return (Path(TOOL_ROOT) / "skills" / "devkb-sdlc" / "SKILL.md").read_text().split("---", 2)[2].strip()
36
+
37
+
38
+ def mcp_config(arm: str, path: Path) -> Path:
39
+ cfg = {"mcpServers": {}}
40
+ if arm in ("graph", "graph_forced"):
41
+ cfg["mcpServers"]["devkb-sdlc"] = {
42
+ "command": "uv", "args": ["run", "--python", ">=3.10", "--no-project", "--with", "mcp>=1.23,<2", "python",
43
+ f"{TOOL_ROOT}/devkb/mcp_server.py", "--project", "ODSF"],
44
+ "env": {"DEVKB_DIR": SNAPSHOT}}
45
+ path.write_text(json.dumps(cfg))
46
+ return path
47
+
48
+
49
+ def run_one(task: dict, arm: str, cfg_dir: Path, tools_extra: list[str] | None = None, cwd: str | None = None, max_turns: int = 30) -> dict:
50
+ cfg = mcp_config(arm, cfg_dir / f"mcp-{arm}.json")
51
+ addendum = (HERE / "control.md").read_text() if arm == "plain" else skill_body()
52
+ tools = READ_TOOLS + (tools_extra or []) + (["mcp__devkb-sdlc"] if arm != "plain" else [])
53
+ prompt = task["prompt"] + (" Use the devkb-sdlc tools first (load them with ToolSearch if they are not yet loaded), "
54
+ "then read only what they name." if arm == "graph_forced" else "")
55
+ cmd = ["claude", "-p", prompt, "--model", MODEL, "--output-format", "stream-json", "--verbose",
56
+ "--max-turns", str(max_turns), "--no-session-persistence", "--mcp-config", str(cfg), "--strict-mcp-config",
57
+ "--append-system-prompt", addendum, "--allowedTools", *tools]
58
+ t0 = time.time()
59
+ proc = subprocess.run(cmd, cwd=cwd or CHECKOUT, capture_output=True, text=True, timeout=900)
60
+ tool_calls = graph_calls = result_bytes = 0
61
+ final: dict = {}
62
+ for line in proc.stdout.splitlines():
63
+ try:
64
+ e = json.loads(line)
65
+ except ValueError:
66
+ continue
67
+ if e.get("type") == "assistant":
68
+ for c in e["message"].get("content", []):
69
+ if c.get("type") == "tool_use":
70
+ tool_calls += 1
71
+ graph_calls += c["name"].startswith("mcp__devkb-sdlc")
72
+ elif e.get("type") == "user" and isinstance(e["message"].get("content"), list):
73
+ for c in e["message"]["content"]:
74
+ if c.get("type") == "tool_result":
75
+ result_bytes += len(json.dumps(c.get("content")))
76
+ elif e.get("type") == "result":
77
+ final = e
78
+ u = final.get("usage", {})
79
+ return {"cost": final.get("total_cost_usd"), "turns": final.get("num_turns"), "secs": round(time.time() - t0, 1),
80
+ "fresh": u.get("input_tokens", 0) + u.get("cache_creation_input_tokens", 0) + u.get("output_tokens", 0),
81
+ "cache_read": u.get("cache_read_input_tokens", 0), "output": u.get("output_tokens", 0),
82
+ "tool_calls": tool_calls, "graph_calls": graph_calls, "result_bytes": result_bytes,
83
+ "is_error": bool(final.get("is_error")) or not final, "answer": final.get("result", "")}
84
+
85
+
86
+ # ── scoring ──────────────────────────────────────────────────────────────────
87
+ def norm(x: str) -> str:
88
+ x = str(x).strip().strip("`").lstrip("./")
89
+ if "::" in x:
90
+ path, name = x.split("::", 1)
91
+ return f"{path}::{name.split('.')[-1].split('(')[0].strip()}"
92
+ return x
93
+
94
+
95
+ def f1(got, want) -> float:
96
+ g, w = {norm(x) for x in (got or [])}, {norm(x) for x in (want or [])}
97
+ if not g and not w:
98
+ return 1.0
99
+ hit = len(g & w)
100
+ if not hit:
101
+ return 0.0
102
+ p, r = hit / len(g), hit / len(w)
103
+ return 2 * p * r / (p + r)
104
+
105
+
106
+ def parse(text: str) -> dict:
107
+ """The first JSON object in the answer. The model may add prose before or after it, and
108
+ prose can contain braces, so try every opening brace rather than the span to the last one."""
109
+ decoder = json.JSONDecoder()
110
+ for m in re.finditer(r"\{", text or ""):
111
+ try:
112
+ obj, _ = decoder.raw_decode(text[m.start():])
113
+ except ValueError:
114
+ continue
115
+ if isinstance(obj, dict) and obj:
116
+ return obj
117
+ return {}
118
+
119
+
120
+ def score(task: dict, answer: str) -> dict:
121
+ a, t, kind = parse(answer), task["truth"], task["kind"]
122
+ if kind == "callers":
123
+ parts = {"callers": f1(a.get("callers"), t["callers"]), "test_files": f1(a.get("test_files"), t["test_files"]),
124
+ "owner": f1(a.get("owner"), t["owner"]), "forbidden": f1(a.get("forbidden_for"), t["forbidden_for"])}
125
+ elif kind == "docs_recall":
126
+ got = [norm(x) for x in (a.get("docs") or [])][:8]
127
+ parts = {"recall": (len({*got} & {norm(x) for x in t["docs"]}) / len(t["docs"])) if t["docs"] else 1.0,
128
+ "exist": (sum(1 for g in got if (Path(CHECKOUT) / g).exists()) / len(got)) if got else 0.0}
129
+ elif kind == "docs_set":
130
+ parts = {"f1": f1(a.get("adrs"), t["adrs"])}
131
+ elif kind == "set":
132
+ parts = {"rules": 1.0 if sorted(a.get("failing_rules") or []) == sorted(t["failing_rules"]) else 0.0}
133
+ elif kind == "review":
134
+ parts = {"changed": f1(a.get("changed"), t["changed"]), "callers": f1(a.get("callers_outside"), t["callers_outside"]),
135
+ "tests": f1(a.get("test_files"), t["test_files"])}
136
+ else:
137
+ parts = {k: 1.0 if str(a.get(k)) == str(v) else 0.0 for k, v in t.items()}
138
+ main = parts.get("recall", sum(parts.values()) / len(parts))
139
+ return {"score": round(main, 3), "parts": {k: round(v, 3) for k, v in parts.items()}}
140
+
141
+
142
+ def main() -> int:
143
+ ap = argparse.ArgumentParser()
144
+ ap.add_argument("tasks")
145
+ ap.add_argument("out")
146
+ ap.add_argument("--reps", type=int, default=5)
147
+ ap.add_argument("--only", default="")
148
+ ap.add_argument("--seed", type=int, default=7)
149
+ ns = ap.parse_args()
150
+ if not (CHECKOUT and SNAPSHOT):
151
+ ap.error("set BENCH_CHECKOUT (pinned worktree) and BENCH_GRAPH (graph snapshot directory)")
152
+ tasks = [t for t in json.load(open(ns.tasks)) if not ns.only or t["stage"] in ns.only.split(",")]
153
+ rnd = random.Random(ns.seed)
154
+ cfg_dir = Path(ns.out).resolve().parent
155
+ pairs = [(t, r) for t in tasks for r in range(ns.reps)]
156
+ rnd.shuffle(pairs)
157
+ with open(ns.out, "a") as out:
158
+ for arm in ("plain", "graph", "graph_forced"): # warm-up: one discarded run per arm so neither pays the cold cache alone
159
+ run_one(tasks[0], arm, cfg_dir)
160
+ for t, rep in pairs:
161
+ for arm in rnd.sample(["plain", "graph", "graph_forced"], 3):
162
+ res = run_one(t, arm, cfg_dir)
163
+ valid = not res["is_error"] and bool(parse(res["answer"]))
164
+ row = {"task": t["id"], "stage": t["stage"], "arm": arm, "rep": rep, "valid": valid,
165
+ **{k: v for k, v in res.items() if k != "answer"},
166
+ **(score(t, res["answer"]) if valid else {"score": 0.0, "parts": {}}), "answer": res["answer"][:1500]}
167
+ out.write(json.dumps(row) + "\n")
168
+ out.flush()
169
+ print(t["id"], arm, rep, "valid" if valid else "INVALID", row["score"], row["cost"], row["turns"], row["graph_calls"], flush=True)
170
+ return 0
171
+
172
+
173
+ if __name__ == "__main__":
174
+ sys.exit(main())
@@ -0,0 +1,148 @@
1
+ """An independent check of each refactor task's truth: the files with a direct caller, found by a second implementation
2
+ that shares no code with the graph (`devkb/build_symbols.py`) or the task generators' truth (`module_truth.py`).
3
+
4
+ The pack check compares a pack with the truth; when both are built with the same import rules, a blind spot they share
5
+ scores 1.0 (lnbits `get_wallet`, 2026-11-15: both missed every caller that imported the function through its package's
6
+ `__init__.py`). This check resolves each import to the file that defines the name, following re-exports, and counts a
7
+ call wherever it is made, inside a function or at import time. A file it finds that the truth lacks is reported as
8
+ `missing`: it is proven by an import chain. A file the truth has that it does not find is `unconfirmed`, weaker
9
+ evidence, since this check does not follow calls through objects.
10
+
11
+ python3 devkb/bench/truth_check.py CHECKOUT TASKS.json
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import ast
16
+ import collections
17
+ import json
18
+ import re
19
+ import subprocess
20
+ import sys
21
+ from pathlib import Path
22
+
23
+ PROMPT = re.compile(r"`([^`]+)` \(defined in ([^)]+)\)")
24
+
25
+
26
+ class _Repo:
27
+ def __init__(self, root: Path) -> None:
28
+ listed = subprocess.run(["git", "ls-files", "--", "*.py"], cwd=root, capture_output=True, text=True).stdout.split()
29
+ self.trees: dict[str, ast.Module] = {}
30
+ for rel in listed:
31
+ try:
32
+ self.trees[rel] = ast.parse((root / rel).read_text(errors="replace"))
33
+ except (SyntaxError, ValueError, OSError, RecursionError):
34
+ continue
35
+ # dotted module -> files whose path ends that way, so `src/` layouts and nested services resolve
36
+ self.by_suffix: dict[str, list[str]] = collections.defaultdict(list)
37
+ for rel in self.trees:
38
+ parts = rel[:-3].split("/")
39
+ if parts[-1] == "__init__":
40
+ parts = parts[:-1]
41
+ for i in range(len(parts)):
42
+ self.by_suffix[".".join(parts[i:])].append(rel)
43
+ self.defined = {rel: {n.name for n in t.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))}
44
+ for rel, t in self.trees.items()}
45
+ self.imports = {rel: self._imports(rel, t) for rel, t in self.trees.items()}
46
+
47
+ def module(self, dotted: str) -> str | None:
48
+ hits = self.by_suffix.get(dotted, [])
49
+ return hits[0] if len(hits) == 1 else None
50
+
51
+ @staticmethod
52
+ def _imports(rel: str, tree: ast.Module) -> tuple[dict[str, tuple[str, str]], dict[str, str]]:
53
+ """({local: (module, name)} from `from m import name`, {local: module} from `import m` / `import m as x`)."""
54
+ names, modules = {}, {}
55
+ pkg = rel[:-3].split("/")[:-1]
56
+ for n in ast.walk(tree):
57
+ if isinstance(n, ast.ImportFrom):
58
+ base = (".".join(pkg[: len(pkg) - n.level + 1] + ([n.module] if n.module else [])) if n.level
59
+ else n.module or "")
60
+ for a in n.names:
61
+ if a.name != "*":
62
+ names[a.asname or a.name] = (base, a.name)
63
+ elif isinstance(n, ast.Import):
64
+ for a in n.names:
65
+ modules[a.asname or a.name] = a.name
66
+ return names, modules
67
+
68
+ def origin(self, dotted: str, name: str, depth: int = 0) -> tuple[str, str] | None:
69
+ """The (file, name) that `name` looked up in module `dotted` is: defined there, or re-exported from elsewhere."""
70
+ rel = self.module(dotted)
71
+ if rel is None: # the package path may be ambiguous while the submodule it names is not
72
+ return ("<module>", f"{dotted}.{name}") if self.module(f"{dotted}.{name}") else None
73
+ if name in self.defined[rel]:
74
+ return rel, name
75
+ via = self.imports[rel][0].get(name)
76
+ if via and depth < 6:
77
+ return self.origin(via[0], via[1], depth + 1)
78
+ sub = f"{dotted}.{name}" # `from pkg import mod`: a module, not a name
79
+ return ("<module>", sub) if self.module(sub) else None
80
+
81
+ def caller_files(self, path: str, name: str) -> set[str]:
82
+ want = (path, name)
83
+ out: set[str] = set()
84
+ for rel, tree in self.trees.items():
85
+ if rel == path: # the defining file calls it by its bare name, from another function
86
+ if any(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name != name and any(
87
+ isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == name for c in ast.walk(n))
88
+ for n in ast.walk(tree)):
89
+ out.add(rel)
90
+ continue
91
+ names, modules = self.imports[rel]
92
+ local = {k for k, (m, n) in names.items() if self.origin(m, n) == want}
93
+ mods: set[str] = set() # local names bound to a module whose `name` is the target
94
+ for k, (m, n) in names.items():
95
+ o = self.origin(m, n)
96
+ if o and o[0] == "<module>" and self.origin(o[1], name) == want:
97
+ mods.add(k)
98
+ for k, m in modules.items():
99
+ if self.origin(m, name) == want:
100
+ mods.add(k)
101
+ for n in ast.walk(tree):
102
+ if not isinstance(n, ast.Call):
103
+ continue
104
+ f = n.func
105
+ if isinstance(f, ast.Name) and f.id in local:
106
+ out.add(rel)
107
+ elif isinstance(f, ast.Attribute) and f.attr == name:
108
+ v, chain = f.value, []
109
+ while isinstance(v, ast.Attribute):
110
+ chain.insert(0, v.attr)
111
+ v = v.value
112
+ if not isinstance(v, ast.Name):
113
+ continue
114
+ if ".".join([v.id, *chain]) in mods:
115
+ out.add(rel)
116
+ continue
117
+ # `models.flow_runs.read_flow_run()`: the root is a module, the rest a path below it
118
+ root = names.get(v.id)
119
+ base = (self.origin(*root) or (None, None)) if root else (None, None)
120
+ dotted = base[1] if base[0] == "<module>" else modules.get(v.id)
121
+ if dotted and chain and self.origin(".".join([dotted, *chain]), name) == want:
122
+ out.add(rel)
123
+ return out
124
+
125
+
126
+ def check(root: Path, tasks: list[dict]) -> list[dict]:
127
+ repo = _Repo(root)
128
+ rows = []
129
+ for t in tasks:
130
+ m = PROMPT.search(t.get("prompt", ""))
131
+ if t.get("kind") != "refactor" or not m or "." in m.group(1):
132
+ continue
133
+ found = repo.caller_files(m.group(2), m.group(1))
134
+ truth = set(t["truth"]["caller_files"])
135
+ rows.append({"task": t["id"], "truth": len(truth), "found": len(found),
136
+ "missing": sorted(found - truth), "unconfirmed": sorted(truth - found)})
137
+ return rows
138
+
139
+
140
+ def main() -> int:
141
+ root, tasks = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text())
142
+ rows = check(root, tasks)
143
+ print(json.dumps(rows, indent=1))
144
+ return 1 if any(r["missing"] for r in rows) else 0
145
+
146
+
147
+ if __name__ == "__main__":
148
+ raise SystemExit(main())
@@ -0,0 +1,64 @@
1
+ #!/usr/bin/env python3
2
+ """Summarise workflow benchmark rows: per task kind and arm, median cost, turns, tool-result bytes, mean score, and the
3
+ cost saved against plain with a bootstrap 95% interval (resampling runs within each task).
4
+
5
+ python3 workflow_report.py results.jsonl
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import collections
10
+ import json
11
+ import random
12
+ import statistics
13
+ import sys
14
+
15
+
16
+ def boot(rows_by_task: dict, arm: str, n: int = 4000) -> tuple[float, float, float]:
17
+ """Cost saved by `arm` against plain: 1 - sum(mean arm cost per task) / sum(mean plain cost per task)."""
18
+ rnd = random.Random(1)
19
+
20
+ def saved(sample):
21
+ a = sum(statistics.mean(s[arm]) for s in sample.values())
22
+ p = sum(statistics.mean(s["plain"]) for s in sample.values())
23
+ return 1 - a / p
24
+
25
+ point = saved(rows_by_task)
26
+ draws = []
27
+ for _ in range(n):
28
+ draws.append(saved({t: {k: [rnd.choice(v) for _ in v] for k, v in s.items()} for t, s in rows_by_task.items()}))
29
+ draws.sort()
30
+ return point, draws[int(n * 0.025)], draws[int(n * 0.975)]
31
+
32
+
33
+ def main() -> int:
34
+ rows = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
35
+ print(f"{len(rows)} runs, {sum(r['valid'] for r in rows)} valid, total ${sum(r['cost'] or 0 for r in rows):.2f}\n")
36
+ kinds = sorted({r["kind"] for r in rows}) + ["ALL"]
37
+ print(f"{'kind':9} {'arm':9} {'n':>2} {'cost$':>7} {'turns':>5} {'bytes':>7} {'score':>5} {'graph%':>6} saved vs plain [95%] tools used")
38
+ for kind in kinds:
39
+ sel = [r for r in rows if kind == "ALL" or r["kind"] == kind]
40
+ by_task: dict = collections.defaultdict(lambda: collections.defaultdict(list))
41
+ for r in sel:
42
+ if r["cost"] is not None:
43
+ by_task[r["task"]][r["arm"]].append(r["cost"])
44
+ arms = [a for a in ("plain", "core", "workflow", "lean", "pack") if any(r["arm"] == a for r in sel)]
45
+ by_task = {t: s for t, s in by_task.items() if all(s.get(a) for a in arms)}
46
+ for arm in arms:
47
+ a = [r for r in sel if r["arm"] == arm]
48
+ if not a:
49
+ continue
50
+ med = lambda k: statistics.median([r[k] or 0 for r in a]) # noqa: E731
51
+ used = collections.Counter(t for r in a for t in r.get("graph_tools", []))
52
+ saved = ""
53
+ if arm != "plain" and by_task:
54
+ p, lo, hi = boot(by_task, arm)
55
+ saved = f"{p:+.0%} [{lo:+.0%}, {hi:+.0%}]"
56
+ print(f"{kind:9} {arm:9} {len(a):2} {med('cost'):7.3f} {med('turns'):5.0f} {med('result_bytes'):7.0f} "
57
+ f"{statistics.mean(r['score'] for r in a):5.2f} {sum(r['graph_calls'] > 0 for r in a) / len(a):6.0%} {saved:22} "
58
+ f"{', '.join(f'{k}×{v}' for k, v in used.most_common(4))}")
59
+ print()
60
+ return 0
61
+
62
+
63
+ if __name__ == "__main__":
64
+ sys.exit(main())
@@ -0,0 +1,206 @@
1
+ #!/usr/bin/env python3
2
+ """Run the workflow benchmark (ODSF-310): whole developer tasks, three arms, interleaved.
3
+
4
+ plain Read, Grep, Glob, read-only git, and a neutral work-efficiently note (control.md)
5
+ core the same plus the devkb-sdlc MCP server with the 11 single-question tools and the devkb-sdlc skill
6
+ workflow the same plus the workflow toolset (one tool per developer task) and the task's own skill
7
+
8
+ Both graph arms are told to use the graph first: ODSF-308 showed that whether the model picks the graph up on its
9
+ own is a separate question (adoption), and this one asks what each tool surface costs when it is used.
10
+
11
+ BENCH_CHECKOUT=... BENCH_GRAPH=... python3 devkb/bench/workflow_run.py TASKS.json OUT.jsonl --reps 4
12
+
13
+ Report: python3 devkb/bench/workflow_report.py OUT.jsonl. Write-up: docs/workflow-benchmark-odsf310.md.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import json
19
+ import os
20
+ import random
21
+ import subprocess
22
+ import sys
23
+ import threading
24
+ import time
25
+ from concurrent.futures import ThreadPoolExecutor
26
+ from pathlib import Path
27
+
28
+ HERE = Path(__file__).resolve().parent
29
+ TOOL_ROOT = Path(os.environ.get("BENCH_TOOL_ROOT", str(HERE.parent.parent))) # checkout holding devkb/mcp_server.py
30
+ # Absolute: the MCP server starts inside the checkout, so a relative graph path would point nowhere and the tool arms
31
+ # would run with no tools while still told to use them (the 2026-11-15 addendum's first attempt).
32
+ CHECKOUT = str(Path(os.environ["BENCH_CHECKOUT"]).resolve()) if os.environ.get("BENCH_CHECKOUT") else "" # pinned worktree
33
+ SNAPSHOT = str(Path(os.environ["BENCH_GRAPH"]).resolve()) if os.environ.get("BENCH_GRAPH") else "" # graph at its HEAD
34
+ MODEL = os.environ.get("BENCH_MODEL", "sonnet")
35
+ PROJECT = os.environ.get("BENCH_PROJECT", "ODSF")
36
+ sys.path.insert(0, str(HERE))
37
+ import run as base # noqa: E402 parse, norm, f1 and READ_TOOLS are shared with the lifecycle benchmark
38
+
39
+ ARMS = tuple(os.environ.get("BENCH_ARMS", "plain,core,workflow").split(",")) # ODSF-313 adds lean and pack
40
+ FORCE = " Use the devkb-sdlc tools first (load them with ToolSearch if they are not yet loaded), then read only what they name."
41
+
42
+
43
+ def skill(name: str) -> str:
44
+ return (TOOL_ROOT / "skills" / name / "SKILL.md").read_text().split("---", 2)[2].strip()
45
+
46
+
47
+ LIMIT_TEXT = ("hit your", "usage limit", "rate limit")
48
+
49
+
50
+ def run_one(task: dict, arm: str, cfg_dir: Path) -> dict:
51
+ """One run. A run the account's usage limit stopped is not a result: wait and run it again (deviation D3). A task
52
+ the agent failed is never retried."""
53
+ for attempt in range(48):
54
+ res = _run_once(task, arm, cfg_dir)
55
+ text = (res.get("answer") or "").lower()
56
+ if not (res.get("is_error") and any(s in text for s in LIMIT_TEXT)):
57
+ res["limit_retries"] = attempt
58
+ return res
59
+ print(f"usage limit on {task['id']} {arm}; waiting 10 minutes (attempt {attempt + 1})", flush=True)
60
+ time.sleep(600)
61
+ return res
62
+
63
+
64
+ def _run_once(task: dict, arm: str, cfg_dir: Path) -> dict:
65
+ cfg = {"mcpServers": {}}
66
+ if arm not in ("plain", "pack"): # a pack arm loads no graph tools: the graph's answer is already in the prompt
67
+ cfg["mcpServers"]["devkb-sdlc"] = {
68
+ # --no-config: uv starts inside the checkout, and a repository's own uv settings (Prefect pins
69
+ # `required-version`) must not stop the graph server; on 2026-11-15 they silently left Prefect's tool arms
70
+ # without tools in all 90 runs.
71
+ "command": "uv", "args": ["run", "--no-config", "--python", ">=3.10", "--no-project", "--with", "mcp>=1.23,<2", "python",
72
+ str(TOOL_ROOT / "devkb" / "mcp_server.py"), "--project", PROJECT],
73
+ "env": {"DEVKB_DIR": SNAPSHOT, "DEVKB_REPO": CHECKOUT, "DEVKB_TOOLSET": arm,
74
+ **({"DEVKB_EXTERNAL": "1"} if os.environ.get("DEVKB_EXTERNAL") == "1" else {}),
75
+ **({"DEVKB_NO_COVERAGE": "1"} if os.environ.get("DEVKB_NO_COVERAGE") == "1" else {})}}
76
+ path = cfg_dir / f"mcp-{arm}-{threading.get_ident()}.json"
77
+ path.write_text(json.dumps(cfg))
78
+ control = (TOOL_ROOT / "devkb" / "bench" / "control.md").read_text()
79
+ addendum = {"plain": control, "pack": control, "core": skill("devkb-sdlc"), "lean": skill("devkb-lean"),
80
+ "lean-pack": skill("devkb-lean-pack"),
81
+ "workflow": skill(task["skill"])}[arm]
82
+ tools = base.READ_TOOLS + (["mcp__devkb-sdlc"] if arm not in ("plain", "pack") else [])
83
+ prompt = task["prompt"] + (FORCE if arm not in ("plain", "pack") else "")
84
+ if arm == "pack":
85
+ prompt += "\n\n" + task["pack"]
86
+ cmd = ["claude", "-p", prompt, "--model", MODEL, "--output-format", "stream-json",
87
+ "--verbose", "--max-turns", os.environ.get("BENCH_MAX_TURNS", "30"), "--no-session-persistence", "--mcp-config", str(path), "--strict-mcp-config",
88
+ "--append-system-prompt", addendum, "--allowedTools", *tools]
89
+ t0 = time.time()
90
+ try:
91
+ proc = subprocess.run(cmd, cwd=CHECKOUT, capture_output=True, text=True, timeout=900)
92
+ stdout = proc.stdout
93
+ except subprocess.TimeoutExpired as exc:
94
+ stdout = exc.stdout.decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "")
95
+ calls, graph, rbytes, used = 0, 0, 0, []
96
+ final: dict = {}
97
+ mcp_ok = arm in ("plain", "pack") # a tool arm counts only when its graph server connected
98
+ contexts: dict[str, int] = {} # message id -> tokens of context that message was given (insertion-ordered)
99
+ for line in stdout.splitlines():
100
+ try:
101
+ e = json.loads(line)
102
+ except ValueError:
103
+ continue
104
+ if e.get("type") == "system" and e.get("subtype") == "init":
105
+ mcp_ok = mcp_ok or any(m.get("name") == "devkb-sdlc" and m.get("status") == "connected"
106
+ for m in e.get("mcp_servers") or [])
107
+ elif e.get("type") == "assistant":
108
+ mu = e["message"].get("usage") or {}
109
+ if e["message"].get("id") and mu:
110
+ contexts[e["message"]["id"]] = (mu.get("input_tokens", 0) + mu.get("cache_creation_input_tokens", 0)
111
+ + mu.get("cache_read_input_tokens", 0))
112
+ for c in e["message"].get("content", []):
113
+ if c.get("type") == "tool_use":
114
+ calls += 1
115
+ if c["name"].startswith("mcp__devkb-sdlc"):
116
+ graph += 1
117
+ used.append(c["name"].split("__")[-1])
118
+ elif e.get("type") == "user" and isinstance(e["message"].get("content"), list):
119
+ rbytes += sum(len(json.dumps(c.get("content"))) for c in e["message"]["content"] if c.get("type") == "tool_result")
120
+ elif e.get("type") == "result":
121
+ final = e
122
+ u = final.get("usage", {})
123
+ cw = u.get("cache_creation") or {}
124
+ ctx = list(contexts.values())
125
+ return {"cost": final.get("total_cost_usd"), "turns": final.get("num_turns"), "secs": round(time.time() - t0, 1),
126
+ "fresh": u.get("input_tokens", 0) + u.get("cache_creation_input_tokens", 0) + u.get("output_tokens", 0),
127
+ "cache_read": u.get("cache_read_input_tokens", 0), "output": u.get("output_tokens", 0),
128
+ # token kinds: uncached input, cache writes (by lifetime), thinking; context size per model call
129
+ "input": u.get("input_tokens", 0), "cache_write": u.get("cache_creation_input_tokens", 0),
130
+ "cache_write_5m": cw.get("ephemeral_5m_input_tokens", 0), "cache_write_1h": cw.get("ephemeral_1h_input_tokens", 0),
131
+ "thinking": (u.get("output_tokens_details") or {}).get("thinking_tokens", 0), "model_calls": len(ctx),
132
+ "ctx_initial": ctx[0] if ctx else 0, "ctx_peak": max(ctx) if ctx else 0, "ctx_final": ctx[-1] if ctx else 0,
133
+ "tool_calls": calls,
134
+ "graph_calls": graph, "graph_tools": used, "result_bytes": rbytes,
135
+ "mcp_connected": mcp_ok,
136
+ "is_error": bool(final.get("is_error")) or not final or not mcp_ok, "answer": final.get("result", "")}
137
+
138
+
139
+ def score(task: dict, answer: str) -> dict:
140
+ a, t, f1, norm = base.parse(answer), task["truth"], base.f1, base.norm
141
+ if task["kind"] == "rca":
142
+ parts = {"raiser": 1.0 if norm(a.get("raiser", "")) == norm(t["raiser"]) else 0.0,
143
+ "callers": f1(a.get("callers"), t["callers"]), "tests": f1(a.get("test_files"), t["test_files"])}
144
+ elif task["kind"] == "err_to_handler":
145
+ parts = {"raiser": 1.0 if norm(a.get("raiser", "")) == norm(t["raiser"]) else 0.0,
146
+ "handlers": f1(a.get("handlers"), t["handlers"]), "tests": f1(a.get("test_files"), t["test_files"])}
147
+ elif task["kind"] == "impact":
148
+ parts = {"handlers": f1(a.get("handlers"), t["handlers"]), "tests": f1(a.get("test_files"), t["test_files"])}
149
+ elif task["kind"] == "ticket_code": # ADDENDUM-1 S2
150
+ parts = {"functions": f1(a.get("functions"), t["functions"]), "tests": f1(a.get("test_files"), t["test_files"])}
151
+ elif task["kind"] == "commit_trace":
152
+ parts = {k: 1.0 if str(a.get(k, "")).strip().upper() == str(t[k]).upper() else 0.0 for k in ("ticket", "epic", "spec")}
153
+ elif task["kind"] == "refactor":
154
+ parts = {"files": f1(a.get("caller_files"), t["caller_files"]), "tests": f1(a.get("test_files"), t["test_files"])}
155
+ if "blocked_files" in t: # ODSF-315: a repository with no role profiles asks no boundary question
156
+ parts["blocked"] = f1(a.get("blocked_files"), t["blocked_files"])
157
+ else: # onboard: how many of the three named are in the true top five (ties at fifth place kept)
158
+ got = [norm(x) for x in (a.get("top") or [])][:3]
159
+ parts = {"p_at_3": sum(1 for g in got if g in {norm(x) for x in t["top5"]}) / 3}
160
+ return {"score": round(sum(parts.values()) / len(parts), 3), "parts": {k: round(v, 3) for k, v in parts.items()}}
161
+
162
+
163
+ def main() -> int:
164
+ ap = argparse.ArgumentParser()
165
+ ap.add_argument("tasks")
166
+ ap.add_argument("out")
167
+ ap.add_argument("--reps", type=int, default=4)
168
+ ap.add_argument("--workers", type=int, default=3)
169
+ ap.add_argument("--seed", type=int, default=310)
170
+ ns = ap.parse_args()
171
+ if not (CHECKOUT and SNAPSHOT):
172
+ ap.error("set BENCH_CHECKOUT (pinned worktree) and BENCH_GRAPH (graph snapshot directory)")
173
+ head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=CHECKOUT, capture_output=True, text=True).stdout.strip()
174
+ built = json.loads((Path(SNAPSHOT) / "manifest.json").read_text()).get("git_head", "")
175
+ if head != built:
176
+ sys.exit(f"graph snapshot {built[:8]} is not the checkout {head[:8]}: the comparison would not be fair")
177
+ tasks = json.load(open(ns.tasks))
178
+ rnd = random.Random(ns.seed)
179
+ cfg_dir = Path(ns.out).resolve().parent
180
+ jobs = [(t, rep, arm) for t in tasks for rep in range(ns.reps) for arm in ARMS]
181
+ rnd.shuffle(jobs)
182
+ for arm in ARMS: # one discarded warm-up per arm, so no arm alone pays the cold prompt cache
183
+ run_one(tasks[0], arm, cfg_dir)
184
+ lock = threading.Lock()
185
+
186
+ def job(j):
187
+ t, rep, arm = j
188
+ res = run_one(t, arm, cfg_dir)
189
+ valid = not res["is_error"] and bool(base.parse(res["answer"]))
190
+ row = {"task": t["id"], "kind": t["kind"], "arm": arm, "rep": rep, "valid": valid, "model": MODEL, "project": PROJECT,
191
+ **{k: v for k, v in res.items() if k != "answer"},
192
+ **(score(t, res["answer"]) if valid else {"score": 0.0, "parts": {}}),
193
+ # the whole answer (capped far above any real one), so a run can be rescored when a truth is corrected;
194
+ # 1,500 characters cut most long file lists in two (lnbits get_wallet, 2026-11-15)
195
+ "answer": res["answer"][:50000]}
196
+ with lock, open(ns.out, "a") as out:
197
+ out.write(json.dumps(row) + "\n")
198
+ print(t["id"], arm, rep, "valid" if valid else "INVALID", row["score"], row["cost"], row["turns"], row["graph_calls"], flush=True)
199
+
200
+ with ThreadPoolExecutor(ns.workers) as pool:
201
+ list(pool.map(job, jobs))
202
+ return 0
203
+
204
+
205
+ if __name__ == "__main__":
206
+ sys.exit(main())