devkb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. devkb/__init__.py +11 -0
  2. devkb/audit.py +173 -0
  3. devkb/bench/add_packs.py +52 -0
  4. devkb/bench/cli_tasks.py +105 -0
  5. devkb/bench/confirm_tasks.py +149 -0
  6. devkb/bench/module_truth.py +249 -0
  7. devkb/bench/oco_tasks.py +216 -0
  8. devkb/bench/pipeline.py +313 -0
  9. devkb/bench/process_tasks.py +135 -0
  10. devkb/bench/report.py +74 -0
  11. devkb/bench/run.py +174 -0
  12. devkb/bench/truth_check.py +148 -0
  13. devkb/bench/workflow_report.py +64 -0
  14. devkb/bench/workflow_run.py +206 -0
  15. devkb/bench/workflow_tasks.py +227 -0
  16. devkb/build.py +258 -0
  17. devkb/build_symbols.py +1394 -0
  18. devkb/call_recall.py +159 -0
  19. devkb/cli.py +53 -0
  20. devkb/coverage.py +155 -0
  21. devkb/edge_truth.py +272 -0
  22. devkb/extractors/backlog.py +113 -0
  23. devkb/extractors/component_subsystem.py +116 -0
  24. devkb/extractors/configuration.py +267 -0
  25. devkb/extractors/files_tests.py +220 -0
  26. devkb/extractors/flows.py +117 -0
  27. devkb/extractors/lifecycle.py +258 -0
  28. devkb/extractors/members.py +160 -0
  29. devkb/extractors/pull_requests.py +271 -0
  30. devkb/extractors/releases.py +49 -0
  31. devkb/extractors/source_baseline.py +66 -0
  32. devkb/extractors/spec_docs.py +220 -0
  33. devkb/extractors/symbol_tests.py +131 -0
  34. devkb/extractors/test_inventory.py +38 -0
  35. devkb/extractors/ticket_similarity.py +193 -0
  36. devkb/extractors/tickets_git.py +151 -0
  37. devkb/extractors/warnings.py +230 -0
  38. devkb/extractors/work_units.py +90 -0
  39. devkb/github.py +515 -0
  40. devkb/gitutil.py +16 -0
  41. devkb/index.py +51 -0
  42. devkb/init.py +178 -0
  43. devkb/mcp_server.py +216 -0
  44. devkb/model.py +221 -0
  45. devkb/module_file_bridge.py +57 -0
  46. devkb/ontology.md +178 -0
  47. devkb/packs.py +462 -0
  48. devkb/process_api.py +1601 -0
  49. devkb/project_context.py +191 -0
  50. devkb/query.py +251 -0
  51. devkb/service.py +324 -0
  52. devkb/startup.py +103 -0
  53. devkb/test_discovery.py +76 -0
  54. devkb/ts_symbols.py +291 -0
  55. devkb-0.2.0.dist-info/METADATA +162 -0
  56. devkb-0.2.0.dist-info/RECORD +61 -0
  57. devkb-0.2.0.dist-info/WHEEL +5 -0
  58. devkb-0.2.0.dist-info/entry_points.txt +2 -0
  59. devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
  60. devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
  61. devkb-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,313 @@
1
+ #!/usr/bin/env python3
2
+ """The benchmark pipeline: one repository from bench/repos.yaml, from a pinned clone to a report.
3
+
4
+ python3 -m devkb.bench.pipeline list
5
+ python3 -m devkb.bench.pipeline prepare ID [--work bench-work] clone, graph, tasks, packs, pack check (no model)
6
+ python3 -m devkb.bench.pipeline run ID --arms plain,core,lean,pack --reps 5 --model sonnet agent runs (paid)
7
+ python3 -m devkb.bench.pipeline report ID cost and quality per arm, against plain
8
+
9
+ prepare is free and deterministic: the same entry gives the same graph, tasks, truth and packs. Its pack check scores
10
+ each pack as if it were an answer, against the truth computed from source, so a repository where the graph is wrong is
11
+ visible before any money is spent. run drives the Claude Code CLI (`claude` on PATH, authenticated) with the same
12
+ harness, arms and protocol as the study: every arm in one shuffled session, a discarded warm-up per arm.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import argparse
17
+ import json
18
+ import os
19
+ import re
20
+ import shutil
21
+ import subprocess
22
+ import sys
23
+ from pathlib import Path
24
+
25
+ import yaml
26
+
27
+ ROOT = Path(__file__).resolve().parents[2]
28
+ REGISTRY = ROOT / "bench" / "repos.yaml"
29
+ BENCH = Path(__file__).resolve().parent
30
+ ID_RX = re.compile(r"^[a-z0-9][a-z0-9-]{0,40}$")
31
+ KINDS = ("web", "cli")
32
+
33
+
34
+ def registry() -> dict[str, dict]:
35
+ entries = (yaml.safe_load(REGISTRY.read_text()) or {}).get("repos", [])
36
+ out = {}
37
+ for e in entries:
38
+ problems = [k for k in ("id", "url", "commit", "kind", "seed", "per_kind") if k not in e]
39
+ if problems:
40
+ raise SystemExit(f"bench/repos.yaml: entry {e.get('id', '?')} lacks {problems}")
41
+ if not ID_RX.match(str(e["id"])) or e["id"] in out:
42
+ raise SystemExit(f"bench/repos.yaml: bad or duplicate id {e['id']!r}")
43
+ if not re.fullmatch(r"[0-9a-f]{40}", str(e["commit"])):
44
+ raise SystemExit(f"bench/repos.yaml: {e['id']}: commit must be a full 40-character hash")
45
+ if e["kind"] not in KINDS:
46
+ raise SystemExit(f"bench/repos.yaml: {e['id']}: kind must be one of {KINDS}")
47
+ if not (str(e["url"]).startswith("https://") or e["url"] == "self"):
48
+ raise SystemExit(f"bench/repos.yaml: {e['id']}: url must be https, or `self` for this repository")
49
+ out[e["id"]] = e
50
+ return out
51
+
52
+
53
+ def sh(*cmd: str, cwd: Path | None = None, env: dict | None = None) -> None:
54
+ print("+", " ".join(cmd), flush=True)
55
+ subprocess.run(cmd, cwd=cwd, check=True, env={**os.environ, **(env or {})})
56
+
57
+
58
+ def paths(work: Path, rid: str) -> dict[str, Path]:
59
+ d = work / rid
60
+ return {"dir": d, "checkout": d / "checkout", "graph": d / "graph", "tasks": d / "tasks.json",
61
+ "packs": d / "tasks.packs.json", "check": d / "pack-check.json", "runs": d / f"{rid}.jsonl",
62
+ "report": d / "report.json"}
63
+
64
+
65
+ def prepare(e: dict, work: Path) -> int:
66
+ p = paths(work, e["id"])
67
+ p["dir"].mkdir(parents=True, exist_ok=True)
68
+ if not (p["checkout"] / ".git").exists():
69
+ sh("git", "init", "-q", str(p["checkout"]))
70
+ # `self`: devkb measured on its own repository, fetched from this checkout (no token needed in CI)
71
+ sh("git", "remote", "add", "origin", str(ROOT) if e["url"] == "self" else e["url"], cwd=p["checkout"])
72
+ sh("git", "fetch", "-q", "origin", e["commit"], cwd=p["checkout"])
73
+ sh("git", "checkout", "-q", "--force", e["commit"], cwd=p["checkout"])
74
+ sh("git", "clean", "-qfdx", cwd=p["checkout"])
75
+ py = sys.executable
76
+ # The graph is built into the snapshot folder, never into the checkout the agents will search.
77
+ env = {"DEVKB_DIR": str(p["graph"])}
78
+ if p["graph"].exists():
79
+ shutil.rmtree(p["graph"])
80
+ sh(py, "-m", "devkb.cli", "build", "--project", str(p["checkout"]), env=env)
81
+ gen = "confirm_tasks.py" if e["kind"] == "web" else "cli_tasks.py"
82
+ sh(py, str(BENCH / gen), str(p["checkout"]), str(p["tasks"]), "--seed", str(e["seed"]),
83
+ "--per-kind", str(e["per_kind"]), "--prefix", f"{e['id']}-")
84
+ tasks = json.loads(p["tasks"].read_text())
85
+ if not tasks:
86
+ print(f"{e['id']}: the generator found no task that meets its selection rules", file=sys.stderr)
87
+ return 1
88
+ sh(py, str(BENCH / "add_packs.py"), str(p["checkout"]), str(p["tasks"]), str(p["packs"]),
89
+ env={"DEVKB_DIR": str(p["graph"]), "DEVKB_REPO": str(p["checkout"])})
90
+ check = pack_check(json.loads(p["packs"].read_text()))
91
+ # The pack check compares the pack with the truth; a blind spot both share scores 1.0. A second implementation
92
+ # of the refactor truth, sharing no code with either, catches that before any run is paid for.
93
+ sys.path.insert(0, str(BENCH))
94
+ import truth_check # noqa: PLC0415
95
+ rows = truth_check.check(p["checkout"], json.loads(p["packs"].read_text()))
96
+ check["truth_check"] = rows
97
+ check["summary"]["truth_missing_callers"] = {r["task"]: r["missing"] for r in rows if r["missing"]}
98
+ p["check"].write_text(json.dumps(check, indent=1))
99
+ print(json.dumps(check["summary"], indent=1))
100
+ if check["summary"]["truth_missing_callers"]:
101
+ print(f"{e['id']}: the truth misses callers an independent check proves (pack-check.json, truth_check); "
102
+ "fix the truth before running agents", file=sys.stderr)
103
+ return 1
104
+ return 0
105
+
106
+
107
+ SECTION = re.compile(r"^## (.+?)(?: \(.*)?$", re.M)
108
+
109
+
110
+ def _sections(pack: str) -> dict[str, list[str]]:
111
+ out: dict[str, list[str]] = {}
112
+ parts = SECTION.split(pack)
113
+ for title, body in zip(parts[1::2], parts[2::2]):
114
+ out[title.strip()] = re.findall(r"^- `([^`]+)`", body, re.M)
115
+ return out
116
+
117
+
118
+ def _short(k: str) -> str:
119
+ return k.split("::")[0] + "::" + k.split("::")[1].split(".")[-1] if "::" in k else k
120
+
121
+
122
+ def pack_check(tasks: list[dict]) -> dict:
123
+ """Each pack read as an answer and scored with the harness's own scorer against the truth from source."""
124
+ sys.path.insert(0, str(BENCH))
125
+ import workflow_run as wr # noqa: PLC0415
126
+
127
+ rows = []
128
+ for t in tasks:
129
+ s = _sections(t["pack"])
130
+ if t["kind"] == "refactor":
131
+ ans = {"caller_files": s.get("Files with a direct caller", []), "test_files": s.get("Test files among them", [])}
132
+ elif t["kind"] in ("rca", "err_to_handler"):
133
+ ans = {"raiser": _short((s.get("Raised by") or [""])[0]),
134
+ "callers": [_short(x) for x in s.get("Functions that reach it, at any depth", [])],
135
+ "handlers": [_short(x) for x in s.get("Route handlers that reach it", [])],
136
+ "test_files": s.get("Test files on those paths", [])}
137
+ elif t["kind"] == "impact":
138
+ ans = {"handlers": [_short(x) for x in s.get("Route handlers that reach it", [])],
139
+ "test_files": [x for x in s.get("Test files to run", [])]}
140
+ else:
141
+ continue
142
+ sc = wr.score(t, json.dumps(ans))
143
+ rows.append({"task": t["id"], "kind": t["kind"], "score": sc["score"], "parts": sc["parts"],
144
+ "complete": "**Complete.**" in t["pack"], "words": len(t["pack"].split())})
145
+ kinds = sorted({r["kind"] for r in rows})
146
+ summary = {"tasks": len(rows), "mean_score": round(sum(r["score"] for r in rows) / max(len(rows), 1), 3),
147
+ "by_kind": {k: round(sum(r["score"] for r in rows if r["kind"] == k) / sum(1 for r in rows if r["kind"] == k), 3) for k in kinds},
148
+ "complete_but_wrong": [r["task"] for r in rows if r["complete"] and r["score"] < 1.0]}
149
+ return {"summary": summary, "tasks": rows}
150
+
151
+
152
+ def run(e: dict, work: Path, arms: str, reps: int, model: str, workers: int) -> int:
153
+ p = paths(work, e["id"])
154
+ if not p["packs"].exists():
155
+ raise SystemExit(f"run `prepare {e['id']}` first")
156
+ if shutil.which("claude") is None:
157
+ raise SystemExit("the Claude Code CLI (`claude`) is not on PATH")
158
+ sh(sys.executable, str(BENCH / "workflow_run.py"), str(p["packs"]), str(p["runs"]), "--reps", str(reps),
159
+ "--workers", str(workers), "--seed", str(e["seed"]),
160
+ env={"BENCH_CHECKOUT": str(p["checkout"]), "BENCH_GRAPH": str(p["graph"]), "BENCH_PROJECT": str(p["checkout"]),
161
+ "BENCH_MODEL": model, "BENCH_ARMS": arms, "BENCH_MAX_TURNS": "40", "BENCH_TOOL_ROOT": str(ROOT)})
162
+ return 0
163
+
164
+
165
+ ARM_ORDER = ("plain", "core", "workflow", "lean", "lean-pack", "pack")
166
+
167
+
168
+ def token_table(rows: list[dict]) -> str:
169
+ """Per arm, the median run's tokens by kind, and the task-weighted change in context processed against plain.
170
+
171
+ input uncached input tokens cache write context written to the prompt cache (first use, or a
172
+ cache read context served from the cache miss or expiry: the context sent again)
173
+ output generated tokens (thinking included) context input + cache write + cache read
174
+ initial ctx context given to the first model call: system prompt, tool definitions, task, pack
175
+ peak ctx the largest context any model call was given
176
+ Runs recorded before these fields existed show input and cache write combined (fresh - output)."""
177
+ import collections # noqa: PLC0415
178
+ import statistics as st # noqa: PLC0415
179
+
180
+ full = all("cache_write" in r for r in rows)
181
+
182
+ def ctx(r):
183
+ return r.get("input", 0) + r.get("cache_write", 0) + r["cache_read"] if "cache_write" in r else r["fresh"] - r["output"] + r["cache_read"]
184
+
185
+ by = collections.defaultdict(lambda: collections.defaultdict(list))
186
+ for r in rows:
187
+ by[r["arm"]][r["task"]].append(r)
188
+ arms = [a for a in ARM_ORDER if a in by]
189
+ tasks = [t for t in by.get("plain", {}) if all(t in by[a] for a in arms)]
190
+ base = sum(st.mean(ctx(r) for r in by["plain"][t]) for t in tasks) if "plain" in by and tasks else 0
191
+ cols = (["cost $", "turns", "model calls", "initial ctx", "peak ctx", "input", "cache write", "cache read", "output",
192
+ "thinking"] if full else ["cost $", "turns", "input + cache write", "cache read", "output"]) + ["context", "context saved vs plain"]
193
+ out = ["| arm | runs | " + " | ".join(cols) + " |", "|---|---:|" + "---:|" * len(cols)]
194
+ for a in arms:
195
+ rs = [r for t in by[a].values() for r in t]
196
+ med = lambda f: st.median(f(r) for r in rs) # noqa: E731
197
+ vals = [f"{med(lambda r: r['cost'] or 0):.4f}", f"{med(lambda r: r['turns'] or 0):g}"]
198
+ if full:
199
+ vals += [f"{med(lambda r: r['model_calls']):g}", f"{med(lambda r: r['ctx_initial']):,.0f}",
200
+ f"{med(lambda r: r['ctx_peak']):,.0f}", f"{med(lambda r: r['input']):,.0f}",
201
+ f"{med(lambda r: r['cache_write']):,.0f}", f"{med(lambda r: r['cache_read']):,.0f}",
202
+ f"{med(lambda r: r['output']):,.0f}", f"{med(lambda r: r['thinking']):,.0f}"]
203
+ else:
204
+ vals += [f"{med(lambda r: r['fresh'] - r['output']):,.0f}", f"{med(lambda r: r['cache_read']):,.0f}",
205
+ f"{med(lambda r: r['output']):,.0f}"]
206
+ vals.append(f"{med(ctx):,.0f}")
207
+ if base and a != "plain":
208
+ vals.append(f"{1 - sum(st.mean(ctx(r) for r in by[a][t]) for t in tasks) / base:+.0%}")
209
+ else:
210
+ vals.append("—")
211
+ out.append(f"| {a} | {len(rs)} | " + " | ".join(vals) + " |")
212
+ return "\n".join(out)
213
+
214
+
215
+ def detail_table(rows: list[dict]) -> str:
216
+ """Per task: median cost, mean score and median turns in each arm; then the mean of each score part per task kind
217
+ and arm (which part of the answer an arm got wrong), and the graph tools each arm called."""
218
+ import collections # noqa: PLC0415
219
+ import statistics as st # noqa: PLC0415
220
+
221
+ by = collections.defaultdict(list)
222
+ for r in rows:
223
+ by[(r["task"], r["arm"])].append(r)
224
+ arms = [a for a in ARM_ORDER if any(r["arm"] == a for r in rows)]
225
+ tasks = sorted({(r["kind"], r["task"]) for r in rows})
226
+ out = ["Per task: median cost $ / mean score / median turns.", "",
227
+ "| kind | task | " + " | ".join(arms) + " |", "|---|---|" + "---|" * len(arms)]
228
+ for kind, t in tasks:
229
+ cells = []
230
+ for a in arms:
231
+ rs = by.get((t, a), [])
232
+ cells.append(f"{st.median((r['cost'] or 0) for r in rs):.3f} / {st.mean(r['score'] for r in rs):.2f} / "
233
+ f"{st.median((r['turns'] or 0) for r in rs):g}" if rs else "—")
234
+ out.append(f"| {kind} | {t} | " + " | ".join(cells) + " |")
235
+ out += ["", "Mean of each score part:", ""]
236
+ for kind in sorted({k for k, _ in tasks}):
237
+ parts = sorted({p for r in rows if r["kind"] == kind for p in (r.get("parts") or {})})
238
+ out += [f"| {kind} | " + " | ".join(parts) + " | runs |", "|---|" + "---:|" * (len(parts) + 1)]
239
+ for a in arms:
240
+ rs = [r for r in rows if r["kind"] == kind and r["arm"] == a]
241
+ if rs:
242
+ out.append(f"| {a} | " + " | ".join(
243
+ f"{st.mean((r.get('parts') or {}).get(p, 0) for r in rs):.2f}" for p in parts) + f" | {len(rs)} |")
244
+ out.append("")
245
+ out += ["Graph tools called (all runs):", ""]
246
+ for a in arms:
247
+ c = collections.Counter(t for r in rows if r["arm"] == a for t in (r.get("graph_tools") or []))
248
+ total = sum((r["cost"] or 0) for r in rows if r["arm"] == a)
249
+ used = ", ".join(f"`{k}` {v}" for k, v in c.most_common()) or "none"
250
+ out.append(f"- **{a}**: ${total:.2f} in all; {used}")
251
+ return "\n".join(out)
252
+
253
+
254
+ def report(e: dict, work: Path, model: str) -> int:
255
+ p = paths(work, e["id"])
256
+ sh(sys.executable, str(ROOT / "study" / "analysis" / "analyze.py"), str(p["runs"]), "--model", model,
257
+ "--out", str(p["report"]))
258
+ r = json.loads(p["report"].read_text())
259
+ print(f"\n{e['id']}: {r['runs']} runs, {r['tasks']} tasks, ${r['cost_usd']}")
260
+ for arm, x in r["pooled"].items():
261
+ c, q = x["cost"], x["quality"]
262
+ print(f" {arm:9s} saved {c['saved']:+.0%} [{c['lo']:+.0%}, {c['hi']:+.0%}] quality {q['diff']:+.3f} "
263
+ f"[{q['lo']:+.3f}, {q['hi']:+.3f}]")
264
+ rows = [json.loads(x) for x in p["runs"].read_text().splitlines() if x.strip()]
265
+ print("\nTokens by kind (median run):\n" + token_table(rows))
266
+ print("\n" + detail_table(rows))
267
+ silent = tool_arms_without_tools(rows)
268
+ if silent:
269
+ print(f"\nWARNING: {', '.join(silent)} never called the graph: their tools were probably unavailable "
270
+ "(check `mcp_connected`); do not report them as tool arms.", file=sys.stderr)
271
+ return 1
272
+ return 0
273
+
274
+
275
+ def tool_arms_without_tools(rows: list[dict]) -> list[str]:
276
+ """Tool arms with runs but not one graph call: Prefect's 90 tool-arm runs of 2026-11-15 ran so, unnoticed."""
277
+ arms = {r["arm"] for r in rows if r["arm"] not in ("plain", "pack")}
278
+ return sorted(a for a in arms if not any(r.get("graph_calls") for r in rows if r["arm"] == a))
279
+
280
+
281
+ def main() -> int:
282
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
283
+ ap.add_argument("command", choices=("list", "prepare", "run", "report", "tokens", "detail"))
284
+ ap.add_argument("id", nargs="?")
285
+ ap.add_argument("--work", default=os.environ.get("BENCH_WORK", "bench-work"))
286
+ ap.add_argument("--arms", default="plain,core,lean,pack")
287
+ ap.add_argument("--reps", type=int, default=5)
288
+ ap.add_argument("--model", default="sonnet")
289
+ ap.add_argument("--workers", type=int, default=3)
290
+ ap.add_argument("--runs", nargs="+", help="tokens: runs files to tabulate")
291
+ a = ap.parse_args()
292
+ if a.command in ("tokens", "detail"):
293
+ rows = [json.loads(x) for f in a.runs for x in Path(f).read_text().splitlines() if x.strip()]
294
+ print(token_table(rows) if a.command == "tokens" else detail_table(rows))
295
+ return 0
296
+ reg = registry()
297
+ if a.command == "list":
298
+ for e in reg.values():
299
+ print(f"{e['id']:16s} {e['kind']:4s} {e['commit'][:10]} {e['url']}")
300
+ return 0
301
+ if a.id not in reg:
302
+ raise SystemExit(f"unknown id {a.id!r}; see bench/repos.yaml")
303
+ work = Path(a.work).resolve()
304
+ e = reg[a.id]
305
+ if a.command == "prepare":
306
+ return prepare(e, work)
307
+ if a.command == "run":
308
+ return run(e, work, a.arms, a.reps, a.model, a.workers)
309
+ return report(e, work, a.model)
310
+
311
+
312
+ if __name__ == "__main__":
313
+ raise SystemExit(main())
@@ -0,0 +1,135 @@
1
+ #!/usr/bin/env python3
2
+ """Process-layer tasks for ADDENDUM-1 S2: joins across tickets, commits, code and tests, truth from git and source.
3
+
4
+ python3 devkb/bench/process_tasks.py CHECKOUT PREFIX OUT.json --seed 2028 --per-kind 4 --tag A-
5
+
6
+ ticket_code for a ticket: the module-level functions its commits changed that still exist at HEAD, and the test
7
+ files that call any of them. Commits: `git log HEAD --grep` on the ticket id, word-anchored. Changed
8
+ functions: functions whose line range at that commit intersects a hunk of the commit's diff. Test
9
+ callers: import-path resolution at HEAD (module_truth.py).
10
+ commit_trace (where tickets record `## Trace`) for a commit: its ticket, the ticket's epic and specification,
11
+ read from the ticket file's Trace table.
12
+
13
+ Selection: tickets whose commits change 2 to 12 such functions, at least one with a test caller; shuffled with the
14
+ seed; up to --per-kind tasks of each kind. Nothing here reads the graph.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import argparse
19
+ import ast
20
+ import json
21
+ import random
22
+ import re
23
+ import subprocess
24
+ import sys
25
+ from pathlib import Path
26
+
27
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
28
+ import module_truth as mt # noqa: E402
29
+
30
+ HUNK = re.compile(r"^@@ -\S+ \+(\d+)(?:,(\d+))? @@", re.M)
31
+
32
+
33
+ def git(root: Path, *args: str) -> str:
34
+ return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True).stdout
35
+
36
+
37
+ def commits_for(root: Path, ticket: str) -> list[str]:
38
+ return git(root, "log", "HEAD", "--format=%H", "-E", f"--grep=(^|[^0-9A-Za-z-]){re.escape(ticket)}([^0-9]|$)").split()
39
+
40
+
41
+ def changed_functions(root: Path, commit: str) -> set[tuple[str, str]]:
42
+ """Module-level functions whose body, at `commit`, intersects a hunk the commit added or changed."""
43
+ out = set()
44
+ files = [f for f in git(root, "show", "--format=", "--name-only", commit).split() if f.endswith(".py")]
45
+ for rel in files:
46
+ if mt.is_test(rel):
47
+ continue
48
+ diff = git(root, "show", "-U0", "--format=", commit, "--", rel)
49
+ spans = []
50
+ for m in HUNK.finditer(diff):
51
+ start, n = int(m.group(1)), int(m.group(2) or 1)
52
+ spans.append((start, start + max(n, 1) - 1))
53
+ if not spans:
54
+ continue
55
+ try:
56
+ tree = ast.parse(git(root, "show", f"{commit}:{rel}"))
57
+ except (SyntaxError, ValueError):
58
+ continue
59
+ for node in tree.body:
60
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
61
+ lo, hi = node.lineno, getattr(node, "end_lineno", node.lineno)
62
+ if any(a <= hi and b >= lo for a, b in spans):
63
+ out.add((rel, node.name))
64
+ return out
65
+
66
+
67
+ def trace(text: str) -> dict:
68
+ rows = dict(re.findall(r"^\| (Epic|Specification) \| (\S+) \|", text, re.M))
69
+ return {"epic": rows.get("Epic"), "spec": rows.get("Specification")}
70
+
71
+
72
+ def main() -> int:
73
+ ap = argparse.ArgumentParser()
74
+ ap.add_argument("checkout")
75
+ ap.add_argument("prefix", help="the ticket prefix, e.g. OCO or ODSF")
76
+ ap.add_argument("out")
77
+ ap.add_argument("--seed", type=int, required=True)
78
+ ap.add_argument("--per-kind", type=int, default=4)
79
+ ap.add_argument("--tag", default="")
80
+ ap.add_argument("--exclude", help="JSON list of names and ticket ids used in exploratory tasks")
81
+ a = ap.parse_args()
82
+ root = Path(a.checkout)
83
+ T = mt.Truth(root)
84
+ callers = T.callers()
85
+ head_functions = {(rel, n) for rel, f in T.files.items() for n in f.functions}
86
+ files = sorted((root / "issues").glob(f"{a.prefix}-*.md"))
87
+ rnd = random.Random(a.seed)
88
+ rnd.shuffle(files)
89
+ tasks, traced = [], []
90
+ exclude = set(json.load(open(a.exclude))) if a.exclude else set()
91
+ id_rx = re.compile(r"\b[A-Z]{2,6}-\d+\b")
92
+ for path in files:
93
+ ticket = re.match(rf"({re.escape(a.prefix)}-\d+)", path.name).group(1)
94
+ if ticket in exclude:
95
+ continue
96
+ commits = commits_for(root, ticket)
97
+ if not commits or len(commits) > 15:
98
+ continue
99
+ changed = set()
100
+ for c in commits:
101
+ changed |= changed_functions(root, c)
102
+ changed &= head_functions
103
+ if not 2 <= len(changed) <= 12:
104
+ continue
105
+ tests = sorted({c[0] for fn in changed for c in callers.get(fn, ()) if mt.is_test(c[0])})
106
+ if tests and len([t for t in tasks if t["kind"] == "ticket_code"]) < a.per_kind:
107
+ tasks.append({"id": f"{a.tag}TCODE-{ticket}", "kind": "ticket_code", "skill": "devkb-sdlc",
108
+ "prompt": (f"Ticket {ticket}: which module-level (top-level, not methods) Python functions did the commits that "
109
+ "reference this ticket change, counting only functions that still exist in the current checkout, "
110
+ "and which test files call any of those functions? Answer with one JSON object: "
111
+ '{"functions": ["path::function", ...], "test_files": ["path", ...]}.'),
112
+ "truth": {"functions": sorted(mt.short(k) for k in changed), "test_files": tests,
113
+ "commits": len(commits)}})
114
+ tr = trace(path.read_text(errors="replace"))
115
+ if tr["epic"] and tr["spec"]:
116
+ # a commit naming one ticket only, so "which ticket does it reference" has one answer
117
+ single = [c for c in commits if set(id_rx.findall(git(root, "log", "-1", "--format=%B", c))) == {ticket}]
118
+ if single:
119
+ traced.append((ticket, single[0], tr))
120
+ if len([t for t in tasks if t["kind"] == "ticket_code"]) >= a.per_kind and len(traced) >= a.per_kind:
121
+ break
122
+ for ticket, commit, tr in traced[: a.per_kind]:
123
+ tasks.append({"id": f"{a.tag}TRACE-{commit[:10]}", "kind": "commit_trace", "skill": "devkb-sdlc",
124
+ "prompt": (f"Commit {commit[:10]}: which ticket does it reference, which epic is that ticket's parent, and which "
125
+ "specification does that ticket implement? Answer with one JSON object: "
126
+ '{"ticket": "...", "epic": "...", "spec": "..."}.'),
127
+ "truth": {"ticket": ticket, "epic": tr["epic"], "spec": tr["spec"]}})
128
+ json.dump(tasks, open(a.out, "w"), indent=1)
129
+ for t in tasks:
130
+ print(t["id"], {k: (len(v) if isinstance(v, list) else v) for k, v in t["truth"].items()})
131
+ return 0
132
+
133
+
134
+ if __name__ == "__main__":
135
+ sys.exit(main())
devkb/bench/report.py ADDED
@@ -0,0 +1,74 @@
1
+ #!/usr/bin/env python3
2
+ """Aggregate lifecycle benchmark results into a per-stage table (ODSF-308).
3
+
4
+ python3 devkb/bench/report.py RESULTS.jsonl [--json]
5
+
6
+ For each stage and arm: valid runs, graph adoption (share of runs that called a graph tool), median cost,
7
+ turns, tokens of tool output returned to the model (bytes), fresh tokens, seconds, and mean correctness.
8
+ Invalid runs (no parsable answer) are counted and excluded from medians; they are never scored as correct.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import statistics
14
+ import sys
15
+ from collections import defaultdict
16
+
17
+ ARMS = ("plain", "graph", "graph_forced")
18
+
19
+
20
+ def med(xs):
21
+ xs = [x for x in xs if x is not None]
22
+ return statistics.median(xs) if xs else None
23
+
24
+
25
+ def pct(a, b):
26
+ return None if a is None or not b else round((a - b) / b * 100)
27
+
28
+
29
+ def aggregate(rows: list[dict]) -> dict:
30
+ cells: dict = defaultdict(list)
31
+ for r in rows:
32
+ cells[(r["stage"], r["arm"])].append(r)
33
+ out: dict = {}
34
+ for (stage, arm), rs in sorted(cells.items()):
35
+ valid = [r for r in rs if r["valid"]]
36
+ out.setdefault(stage, {})[arm] = {
37
+ "runs": len(rs), "valid": len(valid),
38
+ "adoption": round(sum(1 for r in valid if r["graph_calls"] > 0) / len(valid), 2) if valid else None,
39
+ "cost": med([r["cost"] for r in valid]), "turns": med([r["turns"] for r in valid]),
40
+ "bytes": med([r["result_bytes"] for r in valid]), "fresh": med([r["fresh"] for r in valid]),
41
+ "secs": med([r["secs"] for r in valid]),
42
+ "score": round(statistics.mean(r["score"] for r in valid), 2) if valid else None}
43
+ return out
44
+
45
+
46
+ def render(agg: dict) -> str:
47
+ lines = ["| Stage | Arm | Valid | Used graph | Median cost | Turns | Tool output (bytes) | Correct | Cost vs plain | Bytes vs plain |",
48
+ "|---|---|---|---|---|---|---|---|---|---|"]
49
+ for stage, arms in agg.items():
50
+ base = arms.get("plain", {})
51
+ for arm in ARMS:
52
+ c = arms.get(arm)
53
+ if not c:
54
+ continue
55
+ used = "" if c["adoption"] is None else "{:.0%}".format(c["adoption"])
56
+ d_cost = "" if arm == "plain" else "{}%".format(pct(c["cost"], base.get("cost")))
57
+ d_bytes = "" if arm == "plain" else "{}%".format(pct(c["bytes"], base.get("bytes")))
58
+ lines.append("| {} | {} | {}/{} | {} | ${:.3f} | {:.0f} | {:.0f} | {:.2f} | {} | {} |".format(
59
+ stage, arm, c["valid"], c["runs"], used, c["cost"], c["turns"], c["bytes"], c["score"], d_cost, d_bytes))
60
+ totals = {a: sum(arms[a]["cost"] for arms in agg.values() if arms.get(a) and arms[a]["cost"] is not None) for a in ARMS}
61
+ lines.append("")
62
+ lines.append("Sum of stage median costs: " + ", ".join(f"{a} ${v:.2f}" for a, v in totals.items() if v))
63
+ return "\n".join(lines)
64
+
65
+
66
+ def main() -> int:
67
+ rows = [json.loads(line) for line in open(sys.argv[1]) if line.strip()]
68
+ agg = aggregate(rows)
69
+ print(json.dumps(agg, indent=1) if "--json" in sys.argv else render(agg))
70
+ return 0
71
+
72
+
73
+ if __name__ == "__main__":
74
+ sys.exit(main())