devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Workflow benchmark tasks (ODSF-310): whole developer tasks with ground truth from source, never from devkb.
|
|
3
|
+
|
|
4
|
+
Truth is a plain AST scan of the pinned checkout: a call is counted by the called name, and only names defined
|
|
5
|
+
exactly once in the repository are used, so a name-based call is unambiguous. Role boundaries come from the role
|
|
6
|
+
profiles' DO NOT TOUCH blocks, read the way sdp/orchestrator/boundary_check.py reads them.
|
|
7
|
+
|
|
8
|
+
python3 devkb/bench/workflow_tasks.py CHECKOUT OUT.json
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ast
|
|
13
|
+
import collections
|
|
14
|
+
import json
|
|
15
|
+
import random
|
|
16
|
+
import re
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import pathlib
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
SKIP = re.compile(r"(^|/)(\.impl-worktrees|node_modules|\.claude|_bmad)/")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def is_test_file(p: str) -> bool:
|
|
26
|
+
return bool(re.search(r"(^|/)test_[^/]*\.py$|_test\.py$|(^|/)conftest\.py$", p))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Scan(ast.NodeVisitor):
|
|
30
|
+
"""Functions, the names each one calls, and the literal messages each one raises."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, rel: str, out: dict) -> None:
|
|
33
|
+
self.rel, self.out, self.stack, self.fn = rel, out, [], []
|
|
34
|
+
|
|
35
|
+
def visit_ClassDef(self, node):
|
|
36
|
+
self.stack.append(node.name)
|
|
37
|
+
self.generic_visit(node)
|
|
38
|
+
self.stack.pop()
|
|
39
|
+
|
|
40
|
+
def _function(self, node):
|
|
41
|
+
key = (self.rel, ".".join(self.stack + [node.name]))
|
|
42
|
+
self.out["defs"][node.name].append(key)
|
|
43
|
+
self.out["calls"].setdefault(key, set())
|
|
44
|
+
self.out["complexity"][key] = 1 + sum(isinstance(n, (ast.If, ast.For, ast.While, ast.Try, ast.BoolOp, ast.IfExp))
|
|
45
|
+
for n in ast.walk(node))
|
|
46
|
+
self.stack.append(node.name)
|
|
47
|
+
self.fn.append(key)
|
|
48
|
+
self.generic_visit(node)
|
|
49
|
+
self.fn.pop()
|
|
50
|
+
self.stack.pop()
|
|
51
|
+
|
|
52
|
+
visit_FunctionDef = visit_AsyncFunctionDef = _function
|
|
53
|
+
|
|
54
|
+
def visit_Import(self, node):
|
|
55
|
+
for a in node.names:
|
|
56
|
+
self.out["mod_alias"][self.rel][a.asname or a.name.split(".")[0]] = a.name.split(".")[-1] if a.asname else a.name.split(".")[0]
|
|
57
|
+
self.generic_visit(node)
|
|
58
|
+
|
|
59
|
+
def visit_ImportFrom(self, node):
|
|
60
|
+
mod = (node.module or "").split(".")[-1]
|
|
61
|
+
for a in node.names:
|
|
62
|
+
# `from devkb import project_context` binds a module; `from devkb.project_context import resolve` a function
|
|
63
|
+
self.out["from_imp"][self.rel][a.asname or a.name] = (mod, a.name)
|
|
64
|
+
self.generic_visit(node)
|
|
65
|
+
|
|
66
|
+
def visit_Call(self, node):
|
|
67
|
+
if self.fn:
|
|
68
|
+
f = node.func
|
|
69
|
+
if isinstance(f, ast.Name):
|
|
70
|
+
self.out["calls"][self.fn[-1]].add(("name", "", f.id))
|
|
71
|
+
elif isinstance(f, ast.Attribute):
|
|
72
|
+
base = f.value.id if isinstance(f.value, ast.Name) else ""
|
|
73
|
+
self.out["calls"][self.fn[-1]].add(("attr", base, f.attr))
|
|
74
|
+
self.generic_visit(node)
|
|
75
|
+
|
|
76
|
+
def visit_Raise(self, node):
|
|
77
|
+
if self.fn and isinstance(node.exc, ast.Call) and node.exc.args:
|
|
78
|
+
a = node.exc.args[0]
|
|
79
|
+
if isinstance(a, ast.Constant) and isinstance(a.value, str):
|
|
80
|
+
self.out["raises"].append((self.fn[-1], a.value))
|
|
81
|
+
self.generic_visit(node)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def scan(root: Path) -> dict:
|
|
85
|
+
out = {"defs": collections.defaultdict(list), "calls": {}, "raises": [], "complexity": {},
|
|
86
|
+
"mod_alias": collections.defaultdict(dict), "from_imp": collections.defaultdict(dict)}
|
|
87
|
+
files = subprocess.run(["git", "ls-files", "*.py"], cwd=root, capture_output=True, text=True).stdout.split()
|
|
88
|
+
for rel in files:
|
|
89
|
+
if SKIP.search(rel):
|
|
90
|
+
continue
|
|
91
|
+
try:
|
|
92
|
+
Scan(rel, out).visit(ast.parse((root / rel).read_text(errors="replace")))
|
|
93
|
+
except (SyntaxError, ValueError, OSError, RecursionError):
|
|
94
|
+
continue
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# Method names that library objects also have (`Path.resolve`, `dict.get`, `str.split`...): a call `x.resolve()` on an
|
|
99
|
+
# unknown object is not evidence of a call to this repository's `resolve`.
|
|
100
|
+
COLLIDE = set().union(*(dir(t) for t in (str, bytes, list, dict, set, tuple, int, float, object, pathlib.Path)),
|
|
101
|
+
{"run", "get", "read", "write", "close", "update", "add", "send", "start", "stop", "load", "dump",
|
|
102
|
+
"dumps", "loads", "open", "search", "match", "group", "communicate", "wait", "kill", "main"})
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def callers_of(s: dict, name: str) -> list[tuple[str, str]]:
|
|
106
|
+
"""Direct callers of the uniquely named function `name`, resolved through the caller file's imports."""
|
|
107
|
+
def_file = s["defs"][name][0][0]
|
|
108
|
+
stem = Path(def_file).stem
|
|
109
|
+
out = []
|
|
110
|
+
for k, calls in s["calls"].items():
|
|
111
|
+
if k[1].split(".")[-1] == name:
|
|
112
|
+
continue
|
|
113
|
+
f = k[0]
|
|
114
|
+
for kind, base, called in calls:
|
|
115
|
+
if called != name:
|
|
116
|
+
continue
|
|
117
|
+
if kind == "name":
|
|
118
|
+
imp = s["from_imp"][f].get(name)
|
|
119
|
+
if f == def_file or (imp and imp[1] == name and imp[0] in (stem, "")) or name not in COLLIDE and not imp:
|
|
120
|
+
out.append(k)
|
|
121
|
+
break
|
|
122
|
+
else:
|
|
123
|
+
mod = s["mod_alias"][f].get(base) or (s["from_imp"][f].get(base, ("", ""))[1] if base in s["from_imp"][f] else "")
|
|
124
|
+
if mod == stem or (base in ("self", "cls") and f == def_file) or name not in COLLIDE:
|
|
125
|
+
out.append(k)
|
|
126
|
+
break
|
|
127
|
+
return sorted(set(out))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def short(k: tuple[str, str]) -> str:
|
|
131
|
+
return f"{k[0]}::{k[1].split('.')[-1]}"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def dnt(root: Path) -> dict[str, list[str]]:
|
|
135
|
+
if not (root / "pms.config.json").is_file():
|
|
136
|
+
return {} # a repository outside the factory declares no roles (ODSF-315)
|
|
137
|
+
cfg = json.loads((root / "pms.config.json").read_text())
|
|
138
|
+
rx = re.compile(r"### DO NOT TOUCH:\s*```[\w]*\n(.*?)```", re.S)
|
|
139
|
+
out = {}
|
|
140
|
+
profiles = cfg.get("role_profiles") or {}
|
|
141
|
+
if not profiles: # OCO declares a roster and uses the factory's default profile names (project_paths, ODSF-095)
|
|
142
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
143
|
+
from devkb.process_api import ProcessAPI # noqa: PLC0415
|
|
144
|
+
profiles = {c: f for c in cfg.get("team_roles") or [] if (f := ProcessAPI._role_file_in(root, cfg, c))}
|
|
145
|
+
for code, fname in profiles.items():
|
|
146
|
+
if not (root / "sdp" / "process" / "roles" / fname).is_file():
|
|
147
|
+
continue
|
|
148
|
+
m = rx.search((root / "sdp" / "process" / "roles" / fname).read_text())
|
|
149
|
+
out[code] = [c for c in (re.split(r"[←#,]", ln)[0].strip() for ln in (m.group(1).splitlines() if m else [])) if c]
|
|
150
|
+
return out
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def build(root: Path) -> list[dict]:
|
|
154
|
+
s = scan(root)
|
|
155
|
+
unique = {n: ks[0] for n, ks in s["defs"].items() if len(ks) == 1 and not n.startswith("test_") and not n.startswith("__")}
|
|
156
|
+
rnd = random.Random(310)
|
|
157
|
+
tasks: list[dict] = []
|
|
158
|
+
|
|
159
|
+
# RCA: a message raised in exactly one place, by a uniquely named function with real callers and tests.
|
|
160
|
+
counts = collections.Counter(msg for _, msg in s["raises"])
|
|
161
|
+
rca = []
|
|
162
|
+
for key, msg in s["raises"]:
|
|
163
|
+
name = key[1].split(".")[-1]
|
|
164
|
+
if counts[msg] != 1 or len(msg.split()) < 4 or "{" in msg or unique.get(name) != key or is_test_file(key[0]):
|
|
165
|
+
continue
|
|
166
|
+
cs = callers_of(s, name)
|
|
167
|
+
tests = sorted({c[0] for c in cs if is_test_file(c[0])})
|
|
168
|
+
if 2 <= len(cs) <= 10 and tests and any(not is_test_file(c[0]) for c in cs):
|
|
169
|
+
rca.append((key, msg, cs, tests))
|
|
170
|
+
rnd.shuffle(rca)
|
|
171
|
+
for key, msg, cs, tests in rca[:2]:
|
|
172
|
+
name = key[1].split(".")[-1]
|
|
173
|
+
tasks.append({"id": f"RCA-{name}", "stage": "RCA", "kind": "rca", "skill": "devkb-rca",
|
|
174
|
+
"prompt": (f'A user reports this error: "{msg}". Find the root cause location: the function that raises it, every '
|
|
175
|
+
"function that calls that function directly (tests included), and the test files that call it. Answer with one "
|
|
176
|
+
'JSON object: {"raiser": "path::function", "callers": ["path::function", ...], "test_files": ["path", ...]}.'),
|
|
177
|
+
"truth": {"raiser": short(key), "callers": sorted({short(c) for c in cs}), "test_files": tests}})
|
|
178
|
+
|
|
179
|
+
# Refactor scope: a uniquely named function whose callers span several files, for a role that may not edit one of them.
|
|
180
|
+
roles = dnt(root)
|
|
181
|
+
picked = {t["id"].split("-", 1)[1] for t in tasks}
|
|
182
|
+
ref = []
|
|
183
|
+
for name, key in unique.items():
|
|
184
|
+
if name in picked or is_test_file(key[0]) or len(name) < 8:
|
|
185
|
+
continue
|
|
186
|
+
cs = callers_of(s, name)
|
|
187
|
+
files = sorted({c[0] for c in cs})
|
|
188
|
+
if not (3 <= len(files) <= 8 and any(is_test_file(f) for f in files)):
|
|
189
|
+
continue
|
|
190
|
+
for role, prefixes in roles.items():
|
|
191
|
+
blocked = sorted(f for f in files if any(f.startswith(p) for p in prefixes))
|
|
192
|
+
if blocked and len(blocked) < len(files) and not any(key[0].startswith(p) for p in prefixes):
|
|
193
|
+
ref.append((name, key, files, role, blocked))
|
|
194
|
+
break
|
|
195
|
+
rnd.shuffle(ref)
|
|
196
|
+
for name, key, files, role, blocked in ref[:2]:
|
|
197
|
+
tasks.append({"id": f"REF-{name}", "stage": "REFACTOR", "kind": "refactor", "skill": "devkb-refactor",
|
|
198
|
+
"prompt": (f"You work as role {role} and will change the signature of `{name}` (defined in {key[0]}). Before editing, scope the "
|
|
199
|
+
"change: list every file that contains a direct caller (tests included), the test files among them, and which of "
|
|
200
|
+
f"those files role {role} must not edit according to the DO NOT TOUCH block of its profile in sdp/process/roles. "
|
|
201
|
+
'Answer with one JSON object: {"caller_files": [...], "test_files": [...], "blocked_files": [...]}.'),
|
|
202
|
+
"truth": {"caller_files": files, "test_files": [f for f in files if is_test_file(f)], "blocked_files": blocked}})
|
|
203
|
+
|
|
204
|
+
# Onboarding: the most depended-on functions of an area, counting distinct calling functions anywhere.
|
|
205
|
+
for area in ("devkb/", "issue-tools/"):
|
|
206
|
+
rows = []
|
|
207
|
+
for name, key in unique.items():
|
|
208
|
+
if key[0].startswith(area) and not is_test_file(key[0]):
|
|
209
|
+
n = len({c for c in callers_of(s, name)})
|
|
210
|
+
if n:
|
|
211
|
+
rows.append((n, short(key)))
|
|
212
|
+
rows.sort(key=lambda r: (-r[0], r[1]))
|
|
213
|
+
cutoff = rows[4][0] if len(rows) > 4 else 0
|
|
214
|
+
top = [k for n, k in rows if n >= cutoff] # the top five, with ties at the fifth place kept
|
|
215
|
+
tasks.append({"id": f"ONB-{area.strip('/')}", "stage": "ONBOARD", "kind": "onboard", "skill": "devkb-onboard",
|
|
216
|
+
"prompt": (f"You are new to `{area}`. Among the functions defined under `{area}` (excluding test files) whose name is defined "
|
|
217
|
+
"only once in this repository, which three have the most distinct calling functions anywhere in the repository, "
|
|
218
|
+
'tests included? Answer with one JSON object: {"top": ["path::function", "path::function", "path::function"]}.'),
|
|
219
|
+
"truth": {"top3": [k for _, k in rows[:3]], "top5": top, "counts": {k: n for n, k in rows[:8]}}})
|
|
220
|
+
return tasks
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
if __name__ == "__main__":
|
|
224
|
+
t = build(Path(sys.argv[1]))
|
|
225
|
+
json.dump(t, open(sys.argv[2], "w"), indent=1)
|
|
226
|
+
for x in t:
|
|
227
|
+
print(x["id"], {k: (len(v) if isinstance(v, (list, dict)) else v) for k, v in x["truth"].items()})
|
devkb/build.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""devkb build — run every extractor for one project and persist the graph.
|
|
3
|
+
|
|
4
|
+
Usage: python3 devkb/build.py --project CODE
|
|
5
|
+
|
|
6
|
+
Writes <project-repo>/.devkb/{nodes.jsonl, edges.jsonl, manifest.json}. Never
|
|
7
|
+
hand-edit anything under .devkb/ -- it is regenerated wholesale on every run
|
|
8
|
+
(ADR-SDP-TICKETS-001's "generated, disposable index" discipline), so a stale
|
|
9
|
+
run is a rebuild away, never a silent-wrong-answer risk if manifest.json's
|
|
10
|
+
staleness fields are checked by the caller.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import json
|
|
16
|
+
import sys
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
21
|
+
|
|
22
|
+
from devkb import build_symbols
|
|
23
|
+
from devkb.extractors import (
|
|
24
|
+
backlog, component_subsystem, configuration, files_tests, flows, lifecycle, members, releases, source_baseline,
|
|
25
|
+
pull_requests, spec_docs, symbol_tests,
|
|
26
|
+
test_inventory, ticket_similarity, tickets_git, warnings, work_units,
|
|
27
|
+
)
|
|
28
|
+
from devkb.gitutil import git_head
|
|
29
|
+
from devkb.model import DevKBGraph, merge
|
|
30
|
+
from devkb.project_context import resolve
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
#: This checkout — the one whose extractor code is about to run. Distinct from
|
|
34
|
+
#: the target repo, which comes from projects.json.
|
|
35
|
+
BUILDER_ROOT = Path(__file__).resolve().parent.parent
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _roots(path: Path) -> set[str]:
|
|
39
|
+
import subprocess # noqa: PLC0415
|
|
40
|
+
try:
|
|
41
|
+
out = subprocess.run(["git", "rev-list", "--max-parents=0", "HEAD"], cwd=path, capture_output=True,
|
|
42
|
+
text=True, timeout=30).stdout
|
|
43
|
+
except (OSError, subprocess.SubprocessError):
|
|
44
|
+
return set()
|
|
45
|
+
return set(out.split())
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def check_builder_matches_target(repo_root: Path, builder_root: Path = BUILDER_ROOT) -> str | None:
|
|
49
|
+
"""The refusal message when this checkout's code would write another
|
|
50
|
+
checkout's graph, or None when the two agree.
|
|
51
|
+
|
|
52
|
+
ODSF-104. `--project CODE` takes the TARGET from projects.json and the CODE
|
|
53
|
+
from wherever this file happens to live, and those are not required to be
|
|
54
|
+
the same directory. Run a build inside a git worktree and it rebuilds the
|
|
55
|
+
main checkout's graph using the worktree's extractors and the worktree's
|
|
56
|
+
config -- silently. It happened twice in one day: once during the ODSF-102
|
|
57
|
+
landing, and again at 2026-08-16T01:00, when a build from a pre-ODSF-102
|
|
58
|
+
checkout overwrote a correct 5-document spec index with a 2-document one.
|
|
59
|
+
|
|
60
|
+
What makes it hard to catch is that the result looks healthy. The manifest
|
|
61
|
+
records the TARGET's HEAD, so a graph built by stale code carries a current
|
|
62
|
+
commit hash, a recent timestamp and a plausible node count; the only symptom
|
|
63
|
+
is a BA quietly measuring a requirement against two specification documents
|
|
64
|
+
instead of five. A wrong answer wearing a correct label is worse than a
|
|
65
|
+
missing one, so the default is to refuse.
|
|
66
|
+
"""
|
|
67
|
+
if repo_root.resolve() == builder_root.resolve():
|
|
68
|
+
return None
|
|
69
|
+
# Installed on its own, devkb's code is never a checkout of the project it analyses, so there is no stale
|
|
70
|
+
# copy of the project's own extractors to fear. The guard applies only when the builder is another checkout
|
|
71
|
+
# (a worktree, a second clone) of the same repository: the two share a root commit.
|
|
72
|
+
if not (_roots(repo_root) & _roots(builder_root)):
|
|
73
|
+
return None
|
|
74
|
+
return (
|
|
75
|
+
f"this checkout is not the project being built.\n"
|
|
76
|
+
f" extractor code: {builder_root}\n"
|
|
77
|
+
f" target repo: {repo_root}\n"
|
|
78
|
+
f" Building would overwrite the target's graph using THIS checkout's code and\n"
|
|
79
|
+
f" config, which is how a stale index silently replaces a correct one\n"
|
|
80
|
+
f" (ODSF-104). Run it from {repo_root}, or pass --allow-foreign-checkout if\n"
|
|
81
|
+
f" that is genuinely what you want."
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def manifest_payload(
|
|
86
|
+
code: str, repo_root: Path, node_count: int, edge_count: int, github: dict | None = None,
|
|
87
|
+
) -> dict[str, object]:
|
|
88
|
+
"""What `.devkb/manifest.json` records about this build.
|
|
89
|
+
|
|
90
|
+
ODSF-104 added the two `builder_*` fields. `git_head` is the TARGET's, and
|
|
91
|
+
is the same hash whichever checkout's extractors produced the graph — which
|
|
92
|
+
is exactly why a stale build looked current. These say which code ran, so a
|
|
93
|
+
stale graph is identifiable by reading the manifest rather than by noticing
|
|
94
|
+
that a BA answer came out oddly thin.
|
|
95
|
+
|
|
96
|
+
Its own function so the shape can be asserted without building a graph, and
|
|
97
|
+
without a test depending on whether someone has rebuilt recently.
|
|
98
|
+
"""
|
|
99
|
+
payload: dict[str, object] = {
|
|
100
|
+
"project": code,
|
|
101
|
+
"built_at": datetime.now(timezone.utc).isoformat(),
|
|
102
|
+
"git_head": git_head(repo_root),
|
|
103
|
+
"builder_root": str(BUILDER_ROOT),
|
|
104
|
+
"builder_git_head": git_head(BUILDER_ROOT),
|
|
105
|
+
"node_count": node_count,
|
|
106
|
+
"edge_count": edge_count,
|
|
107
|
+
}
|
|
108
|
+
if github is not None:
|
|
109
|
+
# ADR-DEVKB-007 item 6: what the pull-request layer could and could not see, so an answer can say so.
|
|
110
|
+
payload["github"] = github
|
|
111
|
+
return payload
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def build(code: str, allow_foreign_checkout: bool = False, github: bool | None = None,
|
|
115
|
+
github_transport=None) -> tuple[DevKBGraph, Path]:
|
|
116
|
+
"""`github`: True for --github, False for --no-github, None to follow devkb.json (ADR-DEVKB-007: off by default).
|
|
117
|
+
`github_transport` replaces the HTTP client; only tests pass one."""
|
|
118
|
+
ctx = resolve(code)
|
|
119
|
+
mismatch = check_builder_matches_target(ctx.repo_root, BUILDER_ROOT)
|
|
120
|
+
# The guard protects the target's own graph. A build written elsewhere (DEVKB_DIR: a benchmark snapshot) cannot
|
|
121
|
+
# replace it, so another checkout of the same repository may build it.
|
|
122
|
+
if mismatch and not ctx.devkb_dir.resolve().is_relative_to(ctx.repo_root.resolve()):
|
|
123
|
+
mismatch = None
|
|
124
|
+
if mismatch and not allow_foreign_checkout:
|
|
125
|
+
raise ValueError(mismatch)
|
|
126
|
+
if mismatch:
|
|
127
|
+
print(f"devkb build: WARNING — running {BUILDER_ROOT}'s extractors against "
|
|
128
|
+
f"{ctx.repo_root} (--allow-foreign-checkout)", file=sys.stderr)
|
|
129
|
+
tickets_graph = tickets_git.extract(ctx)
|
|
130
|
+
# BL-1: ticket-agnostic, needs only ctx -- runs independently of
|
|
131
|
+
# tickets_git, but BEFORE files_tests so its own TOUCHES data can be
|
|
132
|
+
# reused instead of re-paying git diff-tree for the same commits twice.
|
|
133
|
+
baseline_graph = source_baseline.extract(ctx)
|
|
134
|
+
precomputed_touches: dict[str, list[str]] = {}
|
|
135
|
+
for e in baseline_graph.edges("TOUCHES"):
|
|
136
|
+
precomputed_touches.setdefault(e.src, []).append(e.dst)
|
|
137
|
+
|
|
138
|
+
# ADR-DEVKB-007: pull requests and hosted issues. Offline from git always; GitHub only when asked for.
|
|
139
|
+
pr_graph, github_report = pull_requests.extract(ctx, github, github_transport)
|
|
140
|
+
|
|
141
|
+
# files_tests needs tickets_graph's Commit/Ticket/REFERENCES to derive
|
|
142
|
+
# which commits/tickets to walk -- must run after tickets_git, not
|
|
143
|
+
# interleaved (same ordering rule as merge()'s own docstring).
|
|
144
|
+
graphs = [
|
|
145
|
+
tickets_graph,
|
|
146
|
+
baseline_graph,
|
|
147
|
+
pr_graph,
|
|
148
|
+
files_tests.extract(ctx, tickets_graph, precomputed_touches=precomputed_touches),
|
|
149
|
+
# ODSF-304: a TestCase for every test function, ticket-marked or not.
|
|
150
|
+
test_inventory.extract(ctx),
|
|
151
|
+
backlog.extract(ctx),
|
|
152
|
+
warnings.extract(ctx),
|
|
153
|
+
releases.extract(ctx),
|
|
154
|
+
# BL-8: Component/Subsystem/PART_OF/REALIZES -- needs only ctx
|
|
155
|
+
# (.onboarding/graph.json + pms.config.json), independent of every
|
|
156
|
+
# other extractor here.
|
|
157
|
+
component_subsystem.extract(ctx),
|
|
158
|
+
# ADR-SDP-DEVKB-005 (ODSF-312): ConfigKey/READS/DECLARES and compose/systemd/Dockerfile Components.
|
|
159
|
+
configuration.extract(ctx),
|
|
160
|
+
# BL-3's Symbol graph is a standalone, separately-refreshed artifact
|
|
161
|
+
# (devkb/build_symbols.py, its own manifest) -- this reads whatever
|
|
162
|
+
# was last written there, never regenerates it. Empty if no refresh
|
|
163
|
+
# has run yet, not an error. Carries BL-7/9/10/14 (Endpoint/RAISES/
|
|
164
|
+
# LOGS/REGISTERED_ON/security_signals) along with it, same cache.
|
|
165
|
+
build_symbols.load_cache(ctx),
|
|
166
|
+
]
|
|
167
|
+
graph = merge(graphs)
|
|
168
|
+
|
|
169
|
+
# BL-4/BL-6/BL-11/BL-13 need the FULL merged graph (Symbol/Endpoint
|
|
170
|
+
# nodes from the cache just merged in; Commit/Ticket/REFERENCES/TOUCHES
|
|
171
|
+
# from every extractor above) -- can't run as graphs[] entries, they
|
|
172
|
+
# read `graph` itself. BL-12 (Flow+STEP) runs LAST and deliberately
|
|
173
|
+
# separate from this tuple: it needs the CALLS/CALLS_API graph fully
|
|
174
|
+
# settled, including nothing from BL-11/BL-13 (unrelated) but ordered
|
|
175
|
+
# last anyway per KNOWLEDGE_SPEC.md's own build-order rule.
|
|
176
|
+
for g in (
|
|
177
|
+
symbol_tests.extract(ctx, graph), work_units.extract(ctx, graph),
|
|
178
|
+
ticket_similarity.extract(ctx, graph), spec_docs.extract(ctx, graph),
|
|
179
|
+
):
|
|
180
|
+
for n in g.nodes():
|
|
181
|
+
graph.add_node(n.type, n.natural_key, n.source, **n.props)
|
|
182
|
+
for e in g.edges():
|
|
183
|
+
graph.add_edge(e.verb, e.src_type, e.src, e.dst_type, e.dst, e.source, e.confidence, e.evidence_ref, e.extra)
|
|
184
|
+
|
|
185
|
+
# BL-14 (ADR-SDP-DEVKB-002, ODSF-230): Requirement/Evidence + CHILD_OF/
|
|
186
|
+
# DERIVED_FROM/IMPLEMENTS/EVIDENCED_BY. Its own pass, AFTER the tuple
|
|
187
|
+
# above: IMPLEMENTS resolves a Trace row against spec_docs' Spec nodes and
|
|
188
|
+
# EVIDENCED_BY scans those Spec files, so spec_docs must already be merged.
|
|
189
|
+
lifecycle_graph = lifecycle.extract(ctx, graph)
|
|
190
|
+
for n in lifecycle_graph.nodes():
|
|
191
|
+
graph.add_node(n.type, n.natural_key, n.source, **n.props)
|
|
192
|
+
for e in lifecycle_graph.edges():
|
|
193
|
+
graph.add_edge(e.verb, e.src_type, e.src, e.dst_type, e.dst, e.source, e.confidence, e.evidence_ref, e.extra)
|
|
194
|
+
|
|
195
|
+
flows_graph = flows.extract(ctx, graph)
|
|
196
|
+
for n in flows_graph.nodes():
|
|
197
|
+
graph.add_node(n.type, n.natural_key, n.source, **n.props)
|
|
198
|
+
for e in flows_graph.edges():
|
|
199
|
+
graph.add_edge(e.verb, e.src_type, e.src, e.dst_type, e.dst, e.source, e.confidence, e.evidence_ref, e.extra)
|
|
200
|
+
|
|
201
|
+
# ADR-SDP-DEVKB-005 (ODSF-312): fold in each declared member repository's own graph, then link every File to its
|
|
202
|
+
# Repository. After flows, so a member's code is never walked as if the product called it.
|
|
203
|
+
for row in members.merge_members(ctx, graph, resolve):
|
|
204
|
+
print(f"devkb build: member {row['code']} ({row['role']}): {row['nodes']} nodes, {row['edges']} edges, "
|
|
205
|
+
f"{row['calls_api']} CALLS_API into {code}", file=sys.stderr)
|
|
206
|
+
members.link_repository(graph, code)
|
|
207
|
+
configuration.settle(graph)
|
|
208
|
+
|
|
209
|
+
# Lifecycle needs the FULL merged graph's own Warning nodes (not one
|
|
210
|
+
# extractor's partial view) and has its own history-file side effect,
|
|
211
|
+
# so it runs once, after merge, not as a graphs[] entry.
|
|
212
|
+
warnings.apply_lifecycle(ctx, graph)
|
|
213
|
+
|
|
214
|
+
ctx.devkb_dir.mkdir(parents=True, exist_ok=True)
|
|
215
|
+
data = graph.to_dict()
|
|
216
|
+
|
|
217
|
+
nodes_path = ctx.devkb_dir / "nodes.jsonl"
|
|
218
|
+
edges_path = ctx.devkb_dir / "edges.jsonl"
|
|
219
|
+
manifest_path = ctx.devkb_dir / "manifest.json"
|
|
220
|
+
|
|
221
|
+
nodes_path.write_text("\n".join(json.dumps(n) for n in data["nodes"]) + ("\n" if data["nodes"] else ""))
|
|
222
|
+
edges_path.write_text("\n".join(json.dumps(e) for e in data["edges"]) + ("\n" if data["edges"] else ""))
|
|
223
|
+
manifest_path.write_text(json.dumps(
|
|
224
|
+
manifest_payload(ctx.code, ctx.repo_root, graph.node_count(), graph.edge_count(), github_report),
|
|
225
|
+
indent=2,
|
|
226
|
+
))
|
|
227
|
+
|
|
228
|
+
return graph, ctx.devkb_dir
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def main() -> int:
|
|
232
|
+
ap = argparse.ArgumentParser(description="Build the devkb graph for one project")
|
|
233
|
+
ap.add_argument("--project", required=True, help="Project code, e.g. ODSF (see projects.json)")
|
|
234
|
+
ap.add_argument(
|
|
235
|
+
"--allow-foreign-checkout", action="store_true",
|
|
236
|
+
help="build a project whose repo is not this checkout (ODSF-104: refused by "
|
|
237
|
+
"default, because a build from a worktree silently overwrites the main "
|
|
238
|
+
"graph with that worktree's code)",
|
|
239
|
+
)
|
|
240
|
+
ap.add_argument("--github", dest="github", action="store_true", default=None,
|
|
241
|
+
help="also fetch pull requests and issues from GitHub (ADR-DEVKB-007; opt-in, cached in .devkb/github/)")
|
|
242
|
+
ap.add_argument("--no-github", dest="github", action="store_false",
|
|
243
|
+
help="never contact GitHub, whatever devkb.json says")
|
|
244
|
+
args = ap.parse_args()
|
|
245
|
+
|
|
246
|
+
try:
|
|
247
|
+
graph, devkb_dir = build(args.project, allow_foreign_checkout=args.allow_foreign_checkout, github=args.github)
|
|
248
|
+
except ValueError as exc:
|
|
249
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
250
|
+
return 1
|
|
251
|
+
|
|
252
|
+
print(f"devkb build ({args.project}): {graph.node_count()} nodes, {graph.edge_count()} edges "
|
|
253
|
+
f"-> {devkb_dir}")
|
|
254
|
+
return 0
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
if __name__ == "__main__":
|
|
258
|
+
sys.exit(main())
|