devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/bench/pipeline.py
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""The benchmark pipeline: one repository from bench/repos.yaml, from a pinned clone to a report.
|
|
3
|
+
|
|
4
|
+
python3 -m devkb.bench.pipeline list
|
|
5
|
+
python3 -m devkb.bench.pipeline prepare ID [--work bench-work] clone, graph, tasks, packs, pack check (no model)
|
|
6
|
+
python3 -m devkb.bench.pipeline run ID --arms plain,core,lean,pack --reps 5 --model sonnet agent runs (paid)
|
|
7
|
+
python3 -m devkb.bench.pipeline report ID cost and quality per arm, against plain
|
|
8
|
+
|
|
9
|
+
prepare is free and deterministic: the same entry gives the same graph, tasks, truth and packs. Its pack check scores
|
|
10
|
+
each pack as if it were an answer, against the truth computed from source, so a repository where the graph is wrong is
|
|
11
|
+
visible before any money is spent. run drives the Claude Code CLI (`claude` on PATH, authenticated) with the same
|
|
12
|
+
harness, arms and protocol as the study: every arm in one shuffled session, a discarded warm-up per arm.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import yaml
|
|
26
|
+
|
|
27
|
+
ROOT = Path(__file__).resolve().parents[2]
|
|
28
|
+
REGISTRY = ROOT / "bench" / "repos.yaml"
|
|
29
|
+
BENCH = Path(__file__).resolve().parent
|
|
30
|
+
ID_RX = re.compile(r"^[a-z0-9][a-z0-9-]{0,40}$")
|
|
31
|
+
KINDS = ("web", "cli")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def registry() -> dict[str, dict]:
|
|
35
|
+
entries = (yaml.safe_load(REGISTRY.read_text()) or {}).get("repos", [])
|
|
36
|
+
out = {}
|
|
37
|
+
for e in entries:
|
|
38
|
+
problems = [k for k in ("id", "url", "commit", "kind", "seed", "per_kind") if k not in e]
|
|
39
|
+
if problems:
|
|
40
|
+
raise SystemExit(f"bench/repos.yaml: entry {e.get('id', '?')} lacks {problems}")
|
|
41
|
+
if not ID_RX.match(str(e["id"])) or e["id"] in out:
|
|
42
|
+
raise SystemExit(f"bench/repos.yaml: bad or duplicate id {e['id']!r}")
|
|
43
|
+
if not re.fullmatch(r"[0-9a-f]{40}", str(e["commit"])):
|
|
44
|
+
raise SystemExit(f"bench/repos.yaml: {e['id']}: commit must be a full 40-character hash")
|
|
45
|
+
if e["kind"] not in KINDS:
|
|
46
|
+
raise SystemExit(f"bench/repos.yaml: {e['id']}: kind must be one of {KINDS}")
|
|
47
|
+
if not (str(e["url"]).startswith("https://") or e["url"] == "self"):
|
|
48
|
+
raise SystemExit(f"bench/repos.yaml: {e['id']}: url must be https, or `self` for this repository")
|
|
49
|
+
out[e["id"]] = e
|
|
50
|
+
return out
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def sh(*cmd: str, cwd: Path | None = None, env: dict | None = None) -> None:
|
|
54
|
+
print("+", " ".join(cmd), flush=True)
|
|
55
|
+
subprocess.run(cmd, cwd=cwd, check=True, env={**os.environ, **(env or {})})
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def paths(work: Path, rid: str) -> dict[str, Path]:
|
|
59
|
+
d = work / rid
|
|
60
|
+
return {"dir": d, "checkout": d / "checkout", "graph": d / "graph", "tasks": d / "tasks.json",
|
|
61
|
+
"packs": d / "tasks.packs.json", "check": d / "pack-check.json", "runs": d / f"{rid}.jsonl",
|
|
62
|
+
"report": d / "report.json"}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def prepare(e: dict, work: Path) -> int:
|
|
66
|
+
p = paths(work, e["id"])
|
|
67
|
+
p["dir"].mkdir(parents=True, exist_ok=True)
|
|
68
|
+
if not (p["checkout"] / ".git").exists():
|
|
69
|
+
sh("git", "init", "-q", str(p["checkout"]))
|
|
70
|
+
# `self`: devkb measured on its own repository, fetched from this checkout (no token needed in CI)
|
|
71
|
+
sh("git", "remote", "add", "origin", str(ROOT) if e["url"] == "self" else e["url"], cwd=p["checkout"])
|
|
72
|
+
sh("git", "fetch", "-q", "origin", e["commit"], cwd=p["checkout"])
|
|
73
|
+
sh("git", "checkout", "-q", "--force", e["commit"], cwd=p["checkout"])
|
|
74
|
+
sh("git", "clean", "-qfdx", cwd=p["checkout"])
|
|
75
|
+
py = sys.executable
|
|
76
|
+
# The graph is built into the snapshot folder, never into the checkout the agents will search.
|
|
77
|
+
env = {"DEVKB_DIR": str(p["graph"])}
|
|
78
|
+
if p["graph"].exists():
|
|
79
|
+
shutil.rmtree(p["graph"])
|
|
80
|
+
sh(py, "-m", "devkb.cli", "build", "--project", str(p["checkout"]), env=env)
|
|
81
|
+
gen = "confirm_tasks.py" if e["kind"] == "web" else "cli_tasks.py"
|
|
82
|
+
sh(py, str(BENCH / gen), str(p["checkout"]), str(p["tasks"]), "--seed", str(e["seed"]),
|
|
83
|
+
"--per-kind", str(e["per_kind"]), "--prefix", f"{e['id']}-")
|
|
84
|
+
tasks = json.loads(p["tasks"].read_text())
|
|
85
|
+
if not tasks:
|
|
86
|
+
print(f"{e['id']}: the generator found no task that meets its selection rules", file=sys.stderr)
|
|
87
|
+
return 1
|
|
88
|
+
sh(py, str(BENCH / "add_packs.py"), str(p["checkout"]), str(p["tasks"]), str(p["packs"]),
|
|
89
|
+
env={"DEVKB_DIR": str(p["graph"]), "DEVKB_REPO": str(p["checkout"])})
|
|
90
|
+
check = pack_check(json.loads(p["packs"].read_text()))
|
|
91
|
+
# The pack check compares the pack with the truth; a blind spot both share scores 1.0. A second implementation
|
|
92
|
+
# of the refactor truth, sharing no code with either, catches that before any run is paid for.
|
|
93
|
+
sys.path.insert(0, str(BENCH))
|
|
94
|
+
import truth_check # noqa: PLC0415
|
|
95
|
+
rows = truth_check.check(p["checkout"], json.loads(p["packs"].read_text()))
|
|
96
|
+
check["truth_check"] = rows
|
|
97
|
+
check["summary"]["truth_missing_callers"] = {r["task"]: r["missing"] for r in rows if r["missing"]}
|
|
98
|
+
p["check"].write_text(json.dumps(check, indent=1))
|
|
99
|
+
print(json.dumps(check["summary"], indent=1))
|
|
100
|
+
if check["summary"]["truth_missing_callers"]:
|
|
101
|
+
print(f"{e['id']}: the truth misses callers an independent check proves (pack-check.json, truth_check); "
|
|
102
|
+
"fix the truth before running agents", file=sys.stderr)
|
|
103
|
+
return 1
|
|
104
|
+
return 0
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
SECTION = re.compile(r"^## (.+?)(?: \(.*)?$", re.M)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _sections(pack: str) -> dict[str, list[str]]:
|
|
111
|
+
out: dict[str, list[str]] = {}
|
|
112
|
+
parts = SECTION.split(pack)
|
|
113
|
+
for title, body in zip(parts[1::2], parts[2::2]):
|
|
114
|
+
out[title.strip()] = re.findall(r"^- `([^`]+)`", body, re.M)
|
|
115
|
+
return out
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _short(k: str) -> str:
|
|
119
|
+
return k.split("::")[0] + "::" + k.split("::")[1].split(".")[-1] if "::" in k else k
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def pack_check(tasks: list[dict]) -> dict:
|
|
123
|
+
"""Each pack read as an answer and scored with the harness's own scorer against the truth from source."""
|
|
124
|
+
sys.path.insert(0, str(BENCH))
|
|
125
|
+
import workflow_run as wr # noqa: PLC0415
|
|
126
|
+
|
|
127
|
+
rows = []
|
|
128
|
+
for t in tasks:
|
|
129
|
+
s = _sections(t["pack"])
|
|
130
|
+
if t["kind"] == "refactor":
|
|
131
|
+
ans = {"caller_files": s.get("Files with a direct caller", []), "test_files": s.get("Test files among them", [])}
|
|
132
|
+
elif t["kind"] in ("rca", "err_to_handler"):
|
|
133
|
+
ans = {"raiser": _short((s.get("Raised by") or [""])[0]),
|
|
134
|
+
"callers": [_short(x) for x in s.get("Functions that reach it, at any depth", [])],
|
|
135
|
+
"handlers": [_short(x) for x in s.get("Route handlers that reach it", [])],
|
|
136
|
+
"test_files": s.get("Test files on those paths", [])}
|
|
137
|
+
elif t["kind"] == "impact":
|
|
138
|
+
ans = {"handlers": [_short(x) for x in s.get("Route handlers that reach it", [])],
|
|
139
|
+
"test_files": [x for x in s.get("Test files to run", [])]}
|
|
140
|
+
else:
|
|
141
|
+
continue
|
|
142
|
+
sc = wr.score(t, json.dumps(ans))
|
|
143
|
+
rows.append({"task": t["id"], "kind": t["kind"], "score": sc["score"], "parts": sc["parts"],
|
|
144
|
+
"complete": "**Complete.**" in t["pack"], "words": len(t["pack"].split())})
|
|
145
|
+
kinds = sorted({r["kind"] for r in rows})
|
|
146
|
+
summary = {"tasks": len(rows), "mean_score": round(sum(r["score"] for r in rows) / max(len(rows), 1), 3),
|
|
147
|
+
"by_kind": {k: round(sum(r["score"] for r in rows if r["kind"] == k) / sum(1 for r in rows if r["kind"] == k), 3) for k in kinds},
|
|
148
|
+
"complete_but_wrong": [r["task"] for r in rows if r["complete"] and r["score"] < 1.0]}
|
|
149
|
+
return {"summary": summary, "tasks": rows}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def run(e: dict, work: Path, arms: str, reps: int, model: str, workers: int) -> int:
|
|
153
|
+
p = paths(work, e["id"])
|
|
154
|
+
if not p["packs"].exists():
|
|
155
|
+
raise SystemExit(f"run `prepare {e['id']}` first")
|
|
156
|
+
if shutil.which("claude") is None:
|
|
157
|
+
raise SystemExit("the Claude Code CLI (`claude`) is not on PATH")
|
|
158
|
+
sh(sys.executable, str(BENCH / "workflow_run.py"), str(p["packs"]), str(p["runs"]), "--reps", str(reps),
|
|
159
|
+
"--workers", str(workers), "--seed", str(e["seed"]),
|
|
160
|
+
env={"BENCH_CHECKOUT": str(p["checkout"]), "BENCH_GRAPH": str(p["graph"]), "BENCH_PROJECT": str(p["checkout"]),
|
|
161
|
+
"BENCH_MODEL": model, "BENCH_ARMS": arms, "BENCH_MAX_TURNS": "40", "BENCH_TOOL_ROOT": str(ROOT)})
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
ARM_ORDER = ("plain", "core", "workflow", "lean", "lean-pack", "pack")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def token_table(rows: list[dict]) -> str:
|
|
169
|
+
"""Per arm, the median run's tokens by kind, and the task-weighted change in context processed against plain.
|
|
170
|
+
|
|
171
|
+
input uncached input tokens cache write context written to the prompt cache (first use, or a
|
|
172
|
+
cache read context served from the cache miss or expiry: the context sent again)
|
|
173
|
+
output generated tokens (thinking included) context input + cache write + cache read
|
|
174
|
+
initial ctx context given to the first model call: system prompt, tool definitions, task, pack
|
|
175
|
+
peak ctx the largest context any model call was given
|
|
176
|
+
Runs recorded before these fields existed show input and cache write combined (fresh - output)."""
|
|
177
|
+
import collections # noqa: PLC0415
|
|
178
|
+
import statistics as st # noqa: PLC0415
|
|
179
|
+
|
|
180
|
+
full = all("cache_write" in r for r in rows)
|
|
181
|
+
|
|
182
|
+
def ctx(r):
|
|
183
|
+
return r.get("input", 0) + r.get("cache_write", 0) + r["cache_read"] if "cache_write" in r else r["fresh"] - r["output"] + r["cache_read"]
|
|
184
|
+
|
|
185
|
+
by = collections.defaultdict(lambda: collections.defaultdict(list))
|
|
186
|
+
for r in rows:
|
|
187
|
+
by[r["arm"]][r["task"]].append(r)
|
|
188
|
+
arms = [a for a in ARM_ORDER if a in by]
|
|
189
|
+
tasks = [t for t in by.get("plain", {}) if all(t in by[a] for a in arms)]
|
|
190
|
+
base = sum(st.mean(ctx(r) for r in by["plain"][t]) for t in tasks) if "plain" in by and tasks else 0
|
|
191
|
+
cols = (["cost $", "turns", "model calls", "initial ctx", "peak ctx", "input", "cache write", "cache read", "output",
|
|
192
|
+
"thinking"] if full else ["cost $", "turns", "input + cache write", "cache read", "output"]) + ["context", "context saved vs plain"]
|
|
193
|
+
out = ["| arm | runs | " + " | ".join(cols) + " |", "|---|---:|" + "---:|" * len(cols)]
|
|
194
|
+
for a in arms:
|
|
195
|
+
rs = [r for t in by[a].values() for r in t]
|
|
196
|
+
med = lambda f: st.median(f(r) for r in rs) # noqa: E731
|
|
197
|
+
vals = [f"{med(lambda r: r['cost'] or 0):.4f}", f"{med(lambda r: r['turns'] or 0):g}"]
|
|
198
|
+
if full:
|
|
199
|
+
vals += [f"{med(lambda r: r['model_calls']):g}", f"{med(lambda r: r['ctx_initial']):,.0f}",
|
|
200
|
+
f"{med(lambda r: r['ctx_peak']):,.0f}", f"{med(lambda r: r['input']):,.0f}",
|
|
201
|
+
f"{med(lambda r: r['cache_write']):,.0f}", f"{med(lambda r: r['cache_read']):,.0f}",
|
|
202
|
+
f"{med(lambda r: r['output']):,.0f}", f"{med(lambda r: r['thinking']):,.0f}"]
|
|
203
|
+
else:
|
|
204
|
+
vals += [f"{med(lambda r: r['fresh'] - r['output']):,.0f}", f"{med(lambda r: r['cache_read']):,.0f}",
|
|
205
|
+
f"{med(lambda r: r['output']):,.0f}"]
|
|
206
|
+
vals.append(f"{med(ctx):,.0f}")
|
|
207
|
+
if base and a != "plain":
|
|
208
|
+
vals.append(f"{1 - sum(st.mean(ctx(r) for r in by[a][t]) for t in tasks) / base:+.0%}")
|
|
209
|
+
else:
|
|
210
|
+
vals.append("—")
|
|
211
|
+
out.append(f"| {a} | {len(rs)} | " + " | ".join(vals) + " |")
|
|
212
|
+
return "\n".join(out)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def detail_table(rows: list[dict]) -> str:
|
|
216
|
+
"""Per task: median cost, mean score and median turns in each arm; then the mean of each score part per task kind
|
|
217
|
+
and arm (which part of the answer an arm got wrong), and the graph tools each arm called."""
|
|
218
|
+
import collections # noqa: PLC0415
|
|
219
|
+
import statistics as st # noqa: PLC0415
|
|
220
|
+
|
|
221
|
+
by = collections.defaultdict(list)
|
|
222
|
+
for r in rows:
|
|
223
|
+
by[(r["task"], r["arm"])].append(r)
|
|
224
|
+
arms = [a for a in ARM_ORDER if any(r["arm"] == a for r in rows)]
|
|
225
|
+
tasks = sorted({(r["kind"], r["task"]) for r in rows})
|
|
226
|
+
out = ["Per task: median cost $ / mean score / median turns.", "",
|
|
227
|
+
"| kind | task | " + " | ".join(arms) + " |", "|---|---|" + "---|" * len(arms)]
|
|
228
|
+
for kind, t in tasks:
|
|
229
|
+
cells = []
|
|
230
|
+
for a in arms:
|
|
231
|
+
rs = by.get((t, a), [])
|
|
232
|
+
cells.append(f"{st.median((r['cost'] or 0) for r in rs):.3f} / {st.mean(r['score'] for r in rs):.2f} / "
|
|
233
|
+
f"{st.median((r['turns'] or 0) for r in rs):g}" if rs else "—")
|
|
234
|
+
out.append(f"| {kind} | {t} | " + " | ".join(cells) + " |")
|
|
235
|
+
out += ["", "Mean of each score part:", ""]
|
|
236
|
+
for kind in sorted({k for k, _ in tasks}):
|
|
237
|
+
parts = sorted({p for r in rows if r["kind"] == kind for p in (r.get("parts") or {})})
|
|
238
|
+
out += [f"| {kind} | " + " | ".join(parts) + " | runs |", "|---|" + "---:|" * (len(parts) + 1)]
|
|
239
|
+
for a in arms:
|
|
240
|
+
rs = [r for r in rows if r["kind"] == kind and r["arm"] == a]
|
|
241
|
+
if rs:
|
|
242
|
+
out.append(f"| {a} | " + " | ".join(
|
|
243
|
+
f"{st.mean((r.get('parts') or {}).get(p, 0) for r in rs):.2f}" for p in parts) + f" | {len(rs)} |")
|
|
244
|
+
out.append("")
|
|
245
|
+
out += ["Graph tools called (all runs):", ""]
|
|
246
|
+
for a in arms:
|
|
247
|
+
c = collections.Counter(t for r in rows if r["arm"] == a for t in (r.get("graph_tools") or []))
|
|
248
|
+
total = sum((r["cost"] or 0) for r in rows if r["arm"] == a)
|
|
249
|
+
used = ", ".join(f"`{k}` {v}" for k, v in c.most_common()) or "none"
|
|
250
|
+
out.append(f"- **{a}**: ${total:.2f} in all; {used}")
|
|
251
|
+
return "\n".join(out)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def report(e: dict, work: Path, model: str) -> int:
|
|
255
|
+
p = paths(work, e["id"])
|
|
256
|
+
sh(sys.executable, str(ROOT / "study" / "analysis" / "analyze.py"), str(p["runs"]), "--model", model,
|
|
257
|
+
"--out", str(p["report"]))
|
|
258
|
+
r = json.loads(p["report"].read_text())
|
|
259
|
+
print(f"\n{e['id']}: {r['runs']} runs, {r['tasks']} tasks, ${r['cost_usd']}")
|
|
260
|
+
for arm, x in r["pooled"].items():
|
|
261
|
+
c, q = x["cost"], x["quality"]
|
|
262
|
+
print(f" {arm:9s} saved {c['saved']:+.0%} [{c['lo']:+.0%}, {c['hi']:+.0%}] quality {q['diff']:+.3f} "
|
|
263
|
+
f"[{q['lo']:+.3f}, {q['hi']:+.3f}]")
|
|
264
|
+
rows = [json.loads(x) for x in p["runs"].read_text().splitlines() if x.strip()]
|
|
265
|
+
print("\nTokens by kind (median run):\n" + token_table(rows))
|
|
266
|
+
print("\n" + detail_table(rows))
|
|
267
|
+
silent = tool_arms_without_tools(rows)
|
|
268
|
+
if silent:
|
|
269
|
+
print(f"\nWARNING: {', '.join(silent)} never called the graph: their tools were probably unavailable "
|
|
270
|
+
"(check `mcp_connected`); do not report them as tool arms.", file=sys.stderr)
|
|
271
|
+
return 1
|
|
272
|
+
return 0
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def tool_arms_without_tools(rows: list[dict]) -> list[str]:
|
|
276
|
+
"""Tool arms with runs but not one graph call: Prefect's 90 tool-arm runs of 2026-11-15 ran so, unnoticed."""
|
|
277
|
+
arms = {r["arm"] for r in rows if r["arm"] not in ("plain", "pack")}
|
|
278
|
+
return sorted(a for a in arms if not any(r.get("graph_calls") for r in rows if r["arm"] == a))
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def main() -> int:
|
|
282
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
283
|
+
ap.add_argument("command", choices=("list", "prepare", "run", "report", "tokens", "detail"))
|
|
284
|
+
ap.add_argument("id", nargs="?")
|
|
285
|
+
ap.add_argument("--work", default=os.environ.get("BENCH_WORK", "bench-work"))
|
|
286
|
+
ap.add_argument("--arms", default="plain,core,lean,pack")
|
|
287
|
+
ap.add_argument("--reps", type=int, default=5)
|
|
288
|
+
ap.add_argument("--model", default="sonnet")
|
|
289
|
+
ap.add_argument("--workers", type=int, default=3)
|
|
290
|
+
ap.add_argument("--runs", nargs="+", help="tokens: runs files to tabulate")
|
|
291
|
+
a = ap.parse_args()
|
|
292
|
+
if a.command in ("tokens", "detail"):
|
|
293
|
+
rows = [json.loads(x) for f in a.runs for x in Path(f).read_text().splitlines() if x.strip()]
|
|
294
|
+
print(token_table(rows) if a.command == "tokens" else detail_table(rows))
|
|
295
|
+
return 0
|
|
296
|
+
reg = registry()
|
|
297
|
+
if a.command == "list":
|
|
298
|
+
for e in reg.values():
|
|
299
|
+
print(f"{e['id']:16s} {e['kind']:4s} {e['commit'][:10]} {e['url']}")
|
|
300
|
+
return 0
|
|
301
|
+
if a.id not in reg:
|
|
302
|
+
raise SystemExit(f"unknown id {a.id!r}; see bench/repos.yaml")
|
|
303
|
+
work = Path(a.work).resolve()
|
|
304
|
+
e = reg[a.id]
|
|
305
|
+
if a.command == "prepare":
|
|
306
|
+
return prepare(e, work)
|
|
307
|
+
if a.command == "run":
|
|
308
|
+
return run(e, work, a.arms, a.reps, a.model, a.workers)
|
|
309
|
+
return report(e, work, a.model)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
if __name__ == "__main__":
|
|
313
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Process-layer tasks for ADDENDUM-1 S2: joins across tickets, commits, code and tests, truth from git and source.
|
|
3
|
+
|
|
4
|
+
python3 devkb/bench/process_tasks.py CHECKOUT PREFIX OUT.json --seed 2028 --per-kind 4 --tag A-
|
|
5
|
+
|
|
6
|
+
ticket_code for a ticket: the module-level functions its commits changed that still exist at HEAD, and the test
|
|
7
|
+
files that call any of them. Commits: `git log HEAD --grep` on the ticket id, word-anchored. Changed
|
|
8
|
+
functions: functions whose line range at that commit intersects a hunk of the commit's diff. Test
|
|
9
|
+
callers: import-path resolution at HEAD (module_truth.py).
|
|
10
|
+
commit_trace (where tickets record `## Trace`) for a commit: its ticket, the ticket's epic and specification,
|
|
11
|
+
read from the ticket file's Trace table.
|
|
12
|
+
|
|
13
|
+
Selection: tickets whose commits change 2 to 12 such functions, at least one with a test caller; shuffled with the
|
|
14
|
+
seed; up to --per-kind tasks of each kind. Nothing here reads the graph.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import ast
|
|
20
|
+
import json
|
|
21
|
+
import random
|
|
22
|
+
import re
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
28
|
+
import module_truth as mt # noqa: E402
|
|
29
|
+
|
|
30
|
+
HUNK = re.compile(r"^@@ -\S+ \+(\d+)(?:,(\d+))? @@", re.M)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def git(root: Path, *args: str) -> str:
|
|
34
|
+
return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True).stdout
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def commits_for(root: Path, ticket: str) -> list[str]:
|
|
38
|
+
return git(root, "log", "HEAD", "--format=%H", "-E", f"--grep=(^|[^0-9A-Za-z-]){re.escape(ticket)}([^0-9]|$)").split()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def changed_functions(root: Path, commit: str) -> set[tuple[str, str]]:
|
|
42
|
+
"""Module-level functions whose body, at `commit`, intersects a hunk the commit added or changed."""
|
|
43
|
+
out = set()
|
|
44
|
+
files = [f for f in git(root, "show", "--format=", "--name-only", commit).split() if f.endswith(".py")]
|
|
45
|
+
for rel in files:
|
|
46
|
+
if mt.is_test(rel):
|
|
47
|
+
continue
|
|
48
|
+
diff = git(root, "show", "-U0", "--format=", commit, "--", rel)
|
|
49
|
+
spans = []
|
|
50
|
+
for m in HUNK.finditer(diff):
|
|
51
|
+
start, n = int(m.group(1)), int(m.group(2) or 1)
|
|
52
|
+
spans.append((start, start + max(n, 1) - 1))
|
|
53
|
+
if not spans:
|
|
54
|
+
continue
|
|
55
|
+
try:
|
|
56
|
+
tree = ast.parse(git(root, "show", f"{commit}:{rel}"))
|
|
57
|
+
except (SyntaxError, ValueError):
|
|
58
|
+
continue
|
|
59
|
+
for node in tree.body:
|
|
60
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
61
|
+
lo, hi = node.lineno, getattr(node, "end_lineno", node.lineno)
|
|
62
|
+
if any(a <= hi and b >= lo for a, b in spans):
|
|
63
|
+
out.add((rel, node.name))
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def trace(text: str) -> dict:
|
|
68
|
+
rows = dict(re.findall(r"^\| (Epic|Specification) \| (\S+) \|", text, re.M))
|
|
69
|
+
return {"epic": rows.get("Epic"), "spec": rows.get("Specification")}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def main() -> int:
|
|
73
|
+
ap = argparse.ArgumentParser()
|
|
74
|
+
ap.add_argument("checkout")
|
|
75
|
+
ap.add_argument("prefix", help="the ticket prefix, e.g. OCO or ODSF")
|
|
76
|
+
ap.add_argument("out")
|
|
77
|
+
ap.add_argument("--seed", type=int, required=True)
|
|
78
|
+
ap.add_argument("--per-kind", type=int, default=4)
|
|
79
|
+
ap.add_argument("--tag", default="")
|
|
80
|
+
ap.add_argument("--exclude", help="JSON list of names and ticket ids used in exploratory tasks")
|
|
81
|
+
a = ap.parse_args()
|
|
82
|
+
root = Path(a.checkout)
|
|
83
|
+
T = mt.Truth(root)
|
|
84
|
+
callers = T.callers()
|
|
85
|
+
head_functions = {(rel, n) for rel, f in T.files.items() for n in f.functions}
|
|
86
|
+
files = sorted((root / "issues").glob(f"{a.prefix}-*.md"))
|
|
87
|
+
rnd = random.Random(a.seed)
|
|
88
|
+
rnd.shuffle(files)
|
|
89
|
+
tasks, traced = [], []
|
|
90
|
+
exclude = set(json.load(open(a.exclude))) if a.exclude else set()
|
|
91
|
+
id_rx = re.compile(r"\b[A-Z]{2,6}-\d+\b")
|
|
92
|
+
for path in files:
|
|
93
|
+
ticket = re.match(rf"({re.escape(a.prefix)}-\d+)", path.name).group(1)
|
|
94
|
+
if ticket in exclude:
|
|
95
|
+
continue
|
|
96
|
+
commits = commits_for(root, ticket)
|
|
97
|
+
if not commits or len(commits) > 15:
|
|
98
|
+
continue
|
|
99
|
+
changed = set()
|
|
100
|
+
for c in commits:
|
|
101
|
+
changed |= changed_functions(root, c)
|
|
102
|
+
changed &= head_functions
|
|
103
|
+
if not 2 <= len(changed) <= 12:
|
|
104
|
+
continue
|
|
105
|
+
tests = sorted({c[0] for fn in changed for c in callers.get(fn, ()) if mt.is_test(c[0])})
|
|
106
|
+
if tests and len([t for t in tasks if t["kind"] == "ticket_code"]) < a.per_kind:
|
|
107
|
+
tasks.append({"id": f"{a.tag}TCODE-{ticket}", "kind": "ticket_code", "skill": "devkb-sdlc",
|
|
108
|
+
"prompt": (f"Ticket {ticket}: which module-level (top-level, not methods) Python functions did the commits that "
|
|
109
|
+
"reference this ticket change, counting only functions that still exist in the current checkout, "
|
|
110
|
+
"and which test files call any of those functions? Answer with one JSON object: "
|
|
111
|
+
'{"functions": ["path::function", ...], "test_files": ["path", ...]}.'),
|
|
112
|
+
"truth": {"functions": sorted(mt.short(k) for k in changed), "test_files": tests,
|
|
113
|
+
"commits": len(commits)}})
|
|
114
|
+
tr = trace(path.read_text(errors="replace"))
|
|
115
|
+
if tr["epic"] and tr["spec"]:
|
|
116
|
+
# a commit naming one ticket only, so "which ticket does it reference" has one answer
|
|
117
|
+
single = [c for c in commits if set(id_rx.findall(git(root, "log", "-1", "--format=%B", c))) == {ticket}]
|
|
118
|
+
if single:
|
|
119
|
+
traced.append((ticket, single[0], tr))
|
|
120
|
+
if len([t for t in tasks if t["kind"] == "ticket_code"]) >= a.per_kind and len(traced) >= a.per_kind:
|
|
121
|
+
break
|
|
122
|
+
for ticket, commit, tr in traced[: a.per_kind]:
|
|
123
|
+
tasks.append({"id": f"{a.tag}TRACE-{commit[:10]}", "kind": "commit_trace", "skill": "devkb-sdlc",
|
|
124
|
+
"prompt": (f"Commit {commit[:10]}: which ticket does it reference, which epic is that ticket's parent, and which "
|
|
125
|
+
"specification does that ticket implement? Answer with one JSON object: "
|
|
126
|
+
'{"ticket": "...", "epic": "...", "spec": "..."}.'),
|
|
127
|
+
"truth": {"ticket": ticket, "epic": tr["epic"], "spec": tr["spec"]}})
|
|
128
|
+
json.dump(tasks, open(a.out, "w"), indent=1)
|
|
129
|
+
for t in tasks:
|
|
130
|
+
print(t["id"], {k: (len(v) if isinstance(v, list) else v) for k, v in t["truth"].items()})
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__":
|
|
135
|
+
sys.exit(main())
|
devkb/bench/report.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Aggregate lifecycle benchmark results into a per-stage table (ODSF-308).
|
|
3
|
+
|
|
4
|
+
python3 devkb/bench/report.py RESULTS.jsonl [--json]
|
|
5
|
+
|
|
6
|
+
For each stage and arm: valid runs, graph adoption (share of runs that called a graph tool), median cost,
|
|
7
|
+
turns, tokens of tool output returned to the model (bytes), fresh tokens, seconds, and mean correctness.
|
|
8
|
+
Invalid runs (no parsable answer) are counted and excluded from medians; they are never scored as correct.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import statistics
|
|
14
|
+
import sys
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
|
|
17
|
+
ARMS = ("plain", "graph", "graph_forced")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def med(xs):
|
|
21
|
+
xs = [x for x in xs if x is not None]
|
|
22
|
+
return statistics.median(xs) if xs else None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def pct(a, b):
|
|
26
|
+
return None if a is None or not b else round((a - b) / b * 100)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def aggregate(rows: list[dict]) -> dict:
|
|
30
|
+
cells: dict = defaultdict(list)
|
|
31
|
+
for r in rows:
|
|
32
|
+
cells[(r["stage"], r["arm"])].append(r)
|
|
33
|
+
out: dict = {}
|
|
34
|
+
for (stage, arm), rs in sorted(cells.items()):
|
|
35
|
+
valid = [r for r in rs if r["valid"]]
|
|
36
|
+
out.setdefault(stage, {})[arm] = {
|
|
37
|
+
"runs": len(rs), "valid": len(valid),
|
|
38
|
+
"adoption": round(sum(1 for r in valid if r["graph_calls"] > 0) / len(valid), 2) if valid else None,
|
|
39
|
+
"cost": med([r["cost"] for r in valid]), "turns": med([r["turns"] for r in valid]),
|
|
40
|
+
"bytes": med([r["result_bytes"] for r in valid]), "fresh": med([r["fresh"] for r in valid]),
|
|
41
|
+
"secs": med([r["secs"] for r in valid]),
|
|
42
|
+
"score": round(statistics.mean(r["score"] for r in valid), 2) if valid else None}
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def render(agg: dict) -> str:
|
|
47
|
+
lines = ["| Stage | Arm | Valid | Used graph | Median cost | Turns | Tool output (bytes) | Correct | Cost vs plain | Bytes vs plain |",
|
|
48
|
+
"|---|---|---|---|---|---|---|---|---|---|"]
|
|
49
|
+
for stage, arms in agg.items():
|
|
50
|
+
base = arms.get("plain", {})
|
|
51
|
+
for arm in ARMS:
|
|
52
|
+
c = arms.get(arm)
|
|
53
|
+
if not c:
|
|
54
|
+
continue
|
|
55
|
+
used = "" if c["adoption"] is None else "{:.0%}".format(c["adoption"])
|
|
56
|
+
d_cost = "" if arm == "plain" else "{}%".format(pct(c["cost"], base.get("cost")))
|
|
57
|
+
d_bytes = "" if arm == "plain" else "{}%".format(pct(c["bytes"], base.get("bytes")))
|
|
58
|
+
lines.append("| {} | {} | {}/{} | {} | ${:.3f} | {:.0f} | {:.0f} | {:.2f} | {} | {} |".format(
|
|
59
|
+
stage, arm, c["valid"], c["runs"], used, c["cost"], c["turns"], c["bytes"], c["score"], d_cost, d_bytes))
|
|
60
|
+
totals = {a: sum(arms[a]["cost"] for arms in agg.values() if arms.get(a) and arms[a]["cost"] is not None) for a in ARMS}
|
|
61
|
+
lines.append("")
|
|
62
|
+
lines.append("Sum of stage median costs: " + ", ".join(f"{a} ${v:.2f}" for a, v in totals.items() if v))
|
|
63
|
+
return "\n".join(lines)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main() -> int:
|
|
67
|
+
rows = [json.loads(line) for line in open(sys.argv[1]) if line.strip()]
|
|
68
|
+
agg = aggregate(rows)
|
|
69
|
+
print(json.dumps(agg, indent=1) if "--json" in sys.argv else render(agg))
|
|
70
|
+
return 0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
if __name__ == "__main__":
|
|
74
|
+
sys.exit(main())
|