devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/call_recall.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""How much of the call graph devkb actually holds (ODSF-305).
|
|
3
|
+
|
|
4
|
+
Picks a fixed random sample of functions that are defined exactly once in the
|
|
5
|
+
repository and have 3 to 10 callers and at least 2 tests, finds their true
|
|
6
|
+
callers with an AST scan that shares no code with devkb, then asks the graph
|
|
7
|
+
the same question. Reports caller recall and precision and test-file recall.
|
|
8
|
+
|
|
9
|
+
On 2026-11-13 the graph returned 4 of 36 true callers (11 percent) and none of
|
|
10
|
+
the tests for such a sample, which is why this exists: a graph that answers
|
|
11
|
+
"who calls this?" from a tenth of the answer is worse than no graph.
|
|
12
|
+
|
|
13
|
+
Usage:
|
|
14
|
+
python3 devkb/call_recall.py --project ODSF # report
|
|
15
|
+
python3 devkb/call_recall.py --project ODSF --check # exit 1 below a floor
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import ast
|
|
21
|
+
import collections
|
|
22
|
+
import json
|
|
23
|
+
import random
|
|
24
|
+
import re
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
30
|
+
|
|
31
|
+
from devkb.project_context import resolve # noqa: E402
|
|
32
|
+
|
|
33
|
+
SAMPLE_SIZE = 6
|
|
34
|
+
SEED = 7
|
|
35
|
+
FLOORS = {"caller_recall": 0.8, "caller_precision": 0.9, "test_file_recall": 0.8}
|
|
36
|
+
_SKIP = re.compile(r"(^|/)(\.impl-worktrees|node_modules|\.claude)/")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _Scan(ast.NodeVisitor):
|
|
40
|
+
def __init__(self, rel: str, defs: collections.Counter, calls: dict) -> None:
|
|
41
|
+
self.rel, self.defs, self.calls = rel, defs, calls
|
|
42
|
+
self.stack: list[str] = []
|
|
43
|
+
self.fn: list[tuple[str, str]] = []
|
|
44
|
+
|
|
45
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
|
46
|
+
self.stack.append(node.name)
|
|
47
|
+
self.generic_visit(node)
|
|
48
|
+
self.stack.pop()
|
|
49
|
+
|
|
50
|
+
def _function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
|
|
51
|
+
self.defs[node.name] += 1
|
|
52
|
+
key = (self.rel, ".".join(self.stack + [node.name]))
|
|
53
|
+
self.calls.setdefault(key, set())
|
|
54
|
+
self.fn.append(key)
|
|
55
|
+
self.stack.append(node.name)
|
|
56
|
+
self.generic_visit(node)
|
|
57
|
+
self.stack.pop()
|
|
58
|
+
self.fn.pop()
|
|
59
|
+
|
|
60
|
+
visit_FunctionDef = _function
|
|
61
|
+
visit_AsyncFunctionDef = _function
|
|
62
|
+
|
|
63
|
+
def visit_Call(self, node: ast.Call) -> None:
|
|
64
|
+
if self.fn:
|
|
65
|
+
f = node.func
|
|
66
|
+
name = f.id if isinstance(f, ast.Name) else f.attr if isinstance(f, ast.Attribute) else None
|
|
67
|
+
if name:
|
|
68
|
+
self.calls[self.fn[-1]].add(name)
|
|
69
|
+
self.generic_visit(node)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def truth(repo_root: Path) -> list[dict]:
|
|
73
|
+
files = [f for f in subprocess.run(["git", "ls-files", "--recurse-submodules", "*.py"], cwd=repo_root, capture_output=True,
|
|
74
|
+
text=True, check=False).stdout.split() if not _SKIP.search(f)]
|
|
75
|
+
defs: collections.Counter = collections.Counter()
|
|
76
|
+
calls: dict[tuple[str, str], set[str]] = {}
|
|
77
|
+
for rel in files:
|
|
78
|
+
try:
|
|
79
|
+
_Scan(rel, defs, calls).visit(ast.parse((repo_root / rel).read_text(errors="replace")))
|
|
80
|
+
except (SyntaxError, ValueError, OSError, RecursionError):
|
|
81
|
+
continue
|
|
82
|
+
is_test = lambda k: k[1].split(".")[-1].startswith("test_") # noqa: E731
|
|
83
|
+
pool = []
|
|
84
|
+
for name, count in sorted(defs.items()):
|
|
85
|
+
if count != 1 or len(name) < 9 or name.startswith("__") or name.startswith("test_"):
|
|
86
|
+
continue
|
|
87
|
+
callers = sorted(k for k, s in calls.items() if name in s)
|
|
88
|
+
tests = [k for k in callers if is_test(k)]
|
|
89
|
+
if 3 <= len(callers) <= 10 and 2 <= len(tests) <= 10 and any(not is_test(k) for k in callers):
|
|
90
|
+
pool.append((name, callers))
|
|
91
|
+
random.Random(SEED).shuffle(pool)
|
|
92
|
+
short = lambda k: f"{k[0]}::{k[1].split('.')[-1]}" # noqa: E731
|
|
93
|
+
return [{"function": n, "callers": sorted({short(k) for k in cs}),
|
|
94
|
+
"test_files": sorted({k[0] for k in cs if is_test(k)})} for n, cs in sorted(pool[:SAMPLE_SIZE])]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def graph_answers(devkb_dir: Path, function: str) -> tuple[set[str], set[str]] | None:
|
|
98
|
+
nodes = {}
|
|
99
|
+
by_short: dict[str, list[str]] = collections.defaultdict(list)
|
|
100
|
+
for line in (devkb_dir / "nodes.jsonl").read_text().splitlines():
|
|
101
|
+
n = json.loads(line)
|
|
102
|
+
nodes[n["uid"]] = n
|
|
103
|
+
if n["type"] == "Symbol":
|
|
104
|
+
by_short[n["natural_key"].split("::", 1)[-1].split(".")[-1]].append(n["uid"])
|
|
105
|
+
if len(by_short.get(function, [])) != 1:
|
|
106
|
+
return None
|
|
107
|
+
uid = by_short[function][0]
|
|
108
|
+
callers: set[str] = set()
|
|
109
|
+
tests: set[str] = set()
|
|
110
|
+
for line in (devkb_dir / "edges.jsonl").read_text().splitlines():
|
|
111
|
+
e = json.loads(line)
|
|
112
|
+
if e["verb"] == "CALLS" and e["to_uid"] == uid:
|
|
113
|
+
key = nodes[e["from_uid"]]["natural_key"]
|
|
114
|
+
callers.add(f"{key.split('::')[0]}::{key.split('::')[-1].split('.')[-1]}")
|
|
115
|
+
elif e["verb"] == "TESTED_BY" and e["from_uid"] == uid:
|
|
116
|
+
tests.add(nodes[e["to_uid"]]["natural_key"].split(":")[0])
|
|
117
|
+
return callers, tests
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def measure(code: str) -> dict:
|
|
121
|
+
ctx = resolve(code)
|
|
122
|
+
rows = []
|
|
123
|
+
for q in truth(ctx.repo_root):
|
|
124
|
+
got = graph_answers(ctx.devkb_dir, q["function"])
|
|
125
|
+
callers, tests = got if got else (set(), set())
|
|
126
|
+
want_c, want_t = set(q["callers"]), set(q["test_files"])
|
|
127
|
+
rows.append({"function": q["function"], "true_callers": len(want_c), "graph_callers": len(callers),
|
|
128
|
+
"caller_hits": len(callers & want_c), "true_test_files": len(want_t),
|
|
129
|
+
"test_file_hits": len(tests & want_t)})
|
|
130
|
+
tc, hits = sum(r["true_callers"] for r in rows), sum(r["caller_hits"] for r in rows)
|
|
131
|
+
gc = sum(r["graph_callers"] for r in rows)
|
|
132
|
+
tt, th = sum(r["true_test_files"] for r in rows), sum(r["test_file_hits"] for r in rows)
|
|
133
|
+
result = {"caller_recall": hits / tc if tc else 1.0, "caller_precision": hits / gc if gc else 0.0,
|
|
134
|
+
"test_file_recall": th / tt if tt else 1.0}
|
|
135
|
+
return {"project": code, "rows": rows, "totals": result,
|
|
136
|
+
"ok": all(result[k] >= v for k, v in FLOORS.items())}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def main() -> int:
|
|
140
|
+
ap = argparse.ArgumentParser(description="devkb call-graph recall against an independent AST scan")
|
|
141
|
+
ap.add_argument("--project", required=True)
|
|
142
|
+
ap.add_argument("--check", action="store_true")
|
|
143
|
+
ap.add_argument("--json", action="store_true")
|
|
144
|
+
args = ap.parse_args()
|
|
145
|
+
out = measure(args.project)
|
|
146
|
+
if args.json:
|
|
147
|
+
print(json.dumps(out, indent=2))
|
|
148
|
+
else:
|
|
149
|
+
print(f"devkb call recall ({args.project}), {len(out['rows'])} sampled functions")
|
|
150
|
+
for r in out["rows"]:
|
|
151
|
+
print(f" {r['function']:<32} callers {r['caller_hits']}/{r['true_callers']} (graph returned {r['graph_callers']})"
|
|
152
|
+
f" test files {r['test_file_hits']}/{r['true_test_files']}")
|
|
153
|
+
for k, v in out["totals"].items():
|
|
154
|
+
print(f" {k:<18} {v:.2f} floor {FLOORS[k]:.2f} {'ok' if v >= FLOORS[k] else 'FAIL'}")
|
|
155
|
+
return 1 if args.check and not out["ok"] else 0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
if __name__ == "__main__":
|
|
159
|
+
sys.exit(main())
|
devkb/cli.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""`devkb`: one command for every devkb tool.
|
|
2
|
+
|
|
3
|
+
devkb init [--hooks] [--remove] integrate devkb into this repository (AGENTS.md, CLAUDE.md, .mcp.json)
|
|
4
|
+
devkb build --project . build the graph (symbols, then process) into .devkb/
|
|
5
|
+
devkb pack --project . refactor NAME a pre-computed context pack (error | change | refactor)
|
|
6
|
+
devkb query --project . --about ID the graph's answer about a ticket, file or symbol
|
|
7
|
+
devkb mcp --project . [--toolset lean] serve the graph to an agent over MCP (stdio)
|
|
8
|
+
devkb audit --project . per-relation counts, fan-out and cost of the graph
|
|
9
|
+
devkb coverage --project . what share of the repository the graph covers
|
|
10
|
+
|
|
11
|
+
`--project` takes a repository path (or a code listed in DEVKB_REGISTRY). See devkb/project_context.py.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import runpy
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
COMMANDS = {
|
|
19
|
+
"init": "devkb.init",
|
|
20
|
+
"pack": "devkb.packs",
|
|
21
|
+
"query": "devkb.query",
|
|
22
|
+
"mcp": "devkb.mcp_server",
|
|
23
|
+
"audit": "devkb.audit",
|
|
24
|
+
"coverage": "devkb.coverage",
|
|
25
|
+
"startup": "devkb.startup",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _run(module: str, argv: list[str]) -> int:
|
|
30
|
+
sys.argv = [module, *argv]
|
|
31
|
+
try:
|
|
32
|
+
runpy.run_module(module, run_name="__main__", alter_sys=True)
|
|
33
|
+
except SystemExit as exc:
|
|
34
|
+
return int(exc.code or 0) if not isinstance(exc.code, str) else 1
|
|
35
|
+
return 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main(argv: list[str] | None = None) -> int:
|
|
39
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
40
|
+
if not argv or argv[0] in ("-h", "--help"):
|
|
41
|
+
print(__doc__)
|
|
42
|
+
return 0
|
|
43
|
+
cmd, rest = argv[0], argv[1:]
|
|
44
|
+
if cmd == "build":
|
|
45
|
+
return _run("devkb.build_symbols", rest) or _run("devkb.build", rest)
|
|
46
|
+
if cmd not in COMMANDS:
|
|
47
|
+
print(f"devkb: unknown command {cmd!r}\n{__doc__}", file=sys.stderr)
|
|
48
|
+
return 2
|
|
49
|
+
return _run(COMMANDS[cmd], rest)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
raise SystemExit(main())
|
devkb/coverage.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""How much of a project the devkb graph actually holds (ODSF-304).
|
|
3
|
+
|
|
4
|
+
A graph is only as trustworthy as its coverage, and the gap was invisible:
|
|
5
|
+
One project's graph reported 15,002 nodes for 229 KLOC and nothing said that 94% of
|
|
6
|
+
its test functions and every JavaScript and template file were missing. This
|
|
7
|
+
compares what git tracks with what the graph holds, per category, and fails a
|
|
8
|
+
category below its floor.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
python3 devkb/coverage.py --project . # report
|
|
12
|
+
python3 devkb/coverage.py --project . --check # exit 1 below any floor
|
|
13
|
+
python3 devkb/coverage.py --project . --json
|
|
14
|
+
|
|
15
|
+
Floors default to DEFAULT_FLOORS and are overridden per project in
|
|
16
|
+
`pms.config.json` under `devkb.coverage_floors`. A category with fewer than
|
|
17
|
+
MIN_POPULATION items in the repo is reported but never fails: a project with
|
|
18
|
+
three markdown files has no meaningful specification ratio.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import argparse
|
|
23
|
+
import json
|
|
24
|
+
import re
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
30
|
+
|
|
31
|
+
from devkb.project_context import ProjectContext, load_config, resolve # noqa: E402
|
|
32
|
+
from devkb.test_discovery import list_test_files, tests_in_file # noqa: E402
|
|
33
|
+
|
|
34
|
+
DEFAULT_FLOORS = {
|
|
35
|
+
"test_functions": 0.9,
|
|
36
|
+
"python_files": 0.9,
|
|
37
|
+
"script_files": 0.9,
|
|
38
|
+
"template_files": 0.9,
|
|
39
|
+
"spec_documents": 0.5,
|
|
40
|
+
"ticket_files": 0.9,
|
|
41
|
+
}
|
|
42
|
+
MIN_POPULATION = 10
|
|
43
|
+
|
|
44
|
+
_SCRIPT_SKIP = re.compile(
|
|
45
|
+
r"(\.min\.js$|\.test\.[cm]?[jt]sx?$|\.spec\.[cm]?[jt]sx?$|\.d\.ts$|\.config\.[cm]?[jt]s$|(^|/)fixtures?/"
|
|
46
|
+
r"|(^|/)(lib|libs|vendor|node_modules|output|dist|build|\.impl-worktrees|\.claude)/)")
|
|
47
|
+
# Markdown that is not a specification: tickets, run output, generated reports.
|
|
48
|
+
_DOC_SKIP = re.compile(
|
|
49
|
+
r"(^|/)(issues|historical|output|logs|runtime|node_modules|\.impl-worktrees|\.claude|\.onboarding"
|
|
50
|
+
r"|_bmad-output|sprints|archive|\.agents?)/|(^|/)(README|CHANGELOG|INDEX)\.md$", re.IGNORECASE)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _tracked(repo_root: Path) -> list[str]:
|
|
54
|
+
out = subprocess.run(["git", "ls-files", "--recurse-submodules"], cwd=repo_root, capture_output=True, text=True, check=False).stdout
|
|
55
|
+
return [f for f in out.splitlines() if f]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _load_nodes(ctx: ProjectContext) -> list[dict]:
|
|
59
|
+
path = ctx.devkb_dir / "nodes.jsonl"
|
|
60
|
+
if not path.exists():
|
|
61
|
+
raise SystemExit(f"no graph at {path}; run: python3 devkb/build.py --project {ctx.code}")
|
|
62
|
+
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _floors(ctx: ProjectContext) -> dict[str, float]:
|
|
66
|
+
floors = dict(DEFAULT_FLOORS)
|
|
67
|
+
try:
|
|
68
|
+
cfg = load_config(ctx.repo_root)
|
|
69
|
+
floors.update({k: float(v) for k, v in (cfg.get("devkb", {}).get("coverage_floors") or {}).items()})
|
|
70
|
+
except (OSError, ValueError):
|
|
71
|
+
pass
|
|
72
|
+
return floors
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _row(name: str, covered: int, population: int, floors: dict[str, float], note: str = "") -> dict:
|
|
76
|
+
ratio = covered / population if population else 1.0
|
|
77
|
+
floor = floors[name]
|
|
78
|
+
return {
|
|
79
|
+
"category": name, "covered": covered, "population": population,
|
|
80
|
+
"ratio": round(ratio, 4), "floor": floor, "note": note,
|
|
81
|
+
"enforced": population >= MIN_POPULATION,
|
|
82
|
+
"ok": population < MIN_POPULATION or ratio >= floor,
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def measure(ctx: ProjectContext) -> list[dict]:
|
|
87
|
+
nodes = _load_nodes(ctx)
|
|
88
|
+
floors = _floors(ctx)
|
|
89
|
+
tracked = _tracked(ctx.repo_root)
|
|
90
|
+
rows: list[dict] = []
|
|
91
|
+
|
|
92
|
+
# Test functions: every function discovery finds against TestCase nodes.
|
|
93
|
+
test_keys = {n["natural_key"] for n in nodes if n["type"] == "TestCase"}
|
|
94
|
+
found = 0
|
|
95
|
+
held = 0
|
|
96
|
+
for rel in list_test_files(ctx.repo_root):
|
|
97
|
+
for _name, line in tests_in_file(ctx.repo_root, rel):
|
|
98
|
+
found += 1
|
|
99
|
+
held += f"{rel}:{line}" in test_keys
|
|
100
|
+
rows.append(_row("test_functions", held, found, floors))
|
|
101
|
+
|
|
102
|
+
symbol_files = {n["natural_key"].split("::", 1)[0] for n in nodes if n["type"] == "Symbol"}
|
|
103
|
+
|
|
104
|
+
def files_with_symbols(paths: list[str]) -> int:
|
|
105
|
+
return sum(1 for p in paths if p in symbol_files)
|
|
106
|
+
|
|
107
|
+
py = [f for f in tracked if f.endswith(".py")
|
|
108
|
+
and not re.search(r"(^|/)(test_[^/]+|[^/]+_test|conftest|__init__)\.py$", f)
|
|
109
|
+
and not _SCRIPT_SKIP.search(f)
|
|
110
|
+
and (ctx.repo_root / f).is_file() and (ctx.repo_root / f).stat().st_size > 0]
|
|
111
|
+
rows.append(_row("python_files", files_with_symbols(py), len(py), floors,
|
|
112
|
+
"non-test .py files that define at least one symbol"))
|
|
113
|
+
|
|
114
|
+
scripts = [f for f in tracked if re.search(r"\.(tsx?|[cm]?js)$", f) and not _SCRIPT_SKIP.search(f)]
|
|
115
|
+
rows.append(_row("script_files", files_with_symbols(scripts), len(scripts), floors, "TypeScript and JavaScript"))
|
|
116
|
+
|
|
117
|
+
xml = [f for f in tracked if f.endswith(".xml")
|
|
118
|
+
and re.search(r"(<template\b|<menuitem\b|t-name=|model=\"ir\.(ui\.view|actions\.|cron)|model=\"res\.groups)",
|
|
119
|
+
(ctx.repo_root / f).read_text(errors="replace") if (ctx.repo_root / f).is_file() else "")]
|
|
120
|
+
rows.append(_row("template_files", files_with_symbols(xml), len(xml), floors, "Odoo XML that declares templates, views, actions or menus"))
|
|
121
|
+
|
|
122
|
+
spec_paths = {n["natural_key"] for n in nodes if n["type"] == "Spec"}
|
|
123
|
+
docs = [f for f in tracked if f.endswith(".md") and not _DOC_SKIP.search(f)]
|
|
124
|
+
rows.append(_row("spec_documents", sum(1 for d in docs if d in spec_paths), len(docs), floors,
|
|
125
|
+
"markdown outside issues, history, output and generated folders"))
|
|
126
|
+
|
|
127
|
+
issues_rel = ctx.issues_dir.relative_to(ctx.repo_root).as_posix() if ctx.issues_dir.is_relative_to(ctx.repo_root) else "issues"
|
|
128
|
+
issue_files = [f for f in tracked if f.startswith(issues_rel + "/") and f.endswith(".md") and not f.endswith("README.md")]
|
|
129
|
+
ticket_ids = {n["natural_key"] for n in nodes if n["type"] == "Ticket"}
|
|
130
|
+
held_tickets = sum(1 for f in issue_files if Path(f).name.split("_", 1)[0] in ticket_ids)
|
|
131
|
+
rows.append(_row("ticket_files", held_tickets, len(issue_files), floors))
|
|
132
|
+
return rows
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def main() -> int:
|
|
136
|
+
ap = argparse.ArgumentParser(description="devkb graph coverage against what git tracks")
|
|
137
|
+
ap.add_argument("--project", required=True)
|
|
138
|
+
ap.add_argument("--check", action="store_true", help="exit 1 when any enforced category is below its floor")
|
|
139
|
+
ap.add_argument("--json", action="store_true")
|
|
140
|
+
args = ap.parse_args()
|
|
141
|
+
rows = measure(resolve(args.project))
|
|
142
|
+
if args.json:
|
|
143
|
+
print(json.dumps(rows, indent=2))
|
|
144
|
+
else:
|
|
145
|
+
print(f"devkb coverage ({args.project})")
|
|
146
|
+
for r in rows:
|
|
147
|
+
mark = "ok " if r["ok"] else "FAIL"
|
|
148
|
+
enforced = "" if r["enforced"] else " (not enforced: fewer than %d)" % MIN_POPULATION
|
|
149
|
+
print(f" {mark} {r['category']:<16} {r['covered']:>6} / {r['population']:<6} "
|
|
150
|
+
f"{r['ratio']:>6.1%} floor {r['floor']:.0%}{enforced} {r['note']}")
|
|
151
|
+
return 1 if args.check and any(not r["ok"] for r in rows) else 0
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
sys.exit(main())
|
devkb/edge_truth.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Precision and recall of each edge family, against truth computed from the source, never from devkb (ODSF-314).
|
|
3
|
+
|
|
4
|
+
python3 devkb/edge_truth.py --dir <graph dir> --repo <checkout it was built from> [--json OUT] [--seed 314]
|
|
5
|
+
|
|
6
|
+
Truth per family (sampled where the full set is large, seeded):
|
|
7
|
+
CALLS import-resolved AST callers of uniquely named functions (devkb/bench/workflow_tasks.callers_of),
|
|
8
|
+
recall and precision, and precision per resolution source
|
|
9
|
+
inheritance CALLS resolved only through a base class (source ast-typed-receiver-base): count, and the share whose
|
|
10
|
+
caller's own body calls that method name on something (a necessary, not sufficient, condition)
|
|
11
|
+
TESTED_BY per source: the share of edges whose test function's body calls a function defined in the target
|
|
12
|
+
SIMILAR_TO the share of a ticket's kept neighbours in the same epic or component, against random pairs
|
|
13
|
+
CALLS_API frontend fetch/wrapper URLs (template literals normalised) matched to route paths: file-level recall
|
|
14
|
+
RAISES AST `raise X("literal")` in uniquely named functions: recall
|
|
15
|
+
READS literal os.environ / os.getenv reads in non-test files: file-level recall
|
|
16
|
+
coverage of functions the graph marks complete, the share whose graph callers include every true caller
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import ast
|
|
22
|
+
import collections
|
|
23
|
+
import json
|
|
24
|
+
import random
|
|
25
|
+
import re
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
HERE = Path(__file__).resolve().parent
|
|
31
|
+
sys.path.insert(0, str(HERE / "bench"))
|
|
32
|
+
import workflow_tasks as wt # noqa: E402
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def load(d: Path):
|
|
36
|
+
nodes, by_uid = {}, {}
|
|
37
|
+
for line in (d / "nodes.jsonl").read_text().splitlines():
|
|
38
|
+
if line:
|
|
39
|
+
n = json.loads(line)
|
|
40
|
+
by_uid[n["uid"]] = n
|
|
41
|
+
nodes[(n["type"], n["natural_key"])] = n
|
|
42
|
+
edges = collections.defaultdict(list)
|
|
43
|
+
for line in (d / "edges.jsonl").read_text().splitlines():
|
|
44
|
+
if line:
|
|
45
|
+
e = json.loads(line)
|
|
46
|
+
a, b = by_uid.get(e["from_uid"]), by_uid.get(e["to_uid"])
|
|
47
|
+
if a and b:
|
|
48
|
+
edges[e["verb"]].append((a, b, e.get("properties") or {}))
|
|
49
|
+
return nodes, edges
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def short(key: str) -> str:
|
|
53
|
+
path, _, qual = key.partition("::")
|
|
54
|
+
return f"{path}::{qual.split('.')[-1]}"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def f1(p: float, r: float) -> float:
|
|
58
|
+
return 2 * p * r / (p + r) if p + r else 0.0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def calls_family(s, unique, edges, rnd, sample=200):
|
|
62
|
+
callers_in = collections.defaultdict(set)
|
|
63
|
+
by_src = collections.defaultdict(list)
|
|
64
|
+
for a, b, p in edges["CALLS"]:
|
|
65
|
+
callers_in[short(b["natural_key"])].add(short(a["natural_key"]))
|
|
66
|
+
by_src[p.get("source", "?")].append((short(a["natural_key"]), short(b["natural_key"])))
|
|
67
|
+
pool = [(n, k) for n, k in unique.items() if not wt.is_test_file(k[0]) and wt.callers_of(s, n)]
|
|
68
|
+
rnd.shuffle(pool)
|
|
69
|
+
tp = fn = fp = 0
|
|
70
|
+
truth_pairs = set()
|
|
71
|
+
complete_flags = []
|
|
72
|
+
for name, key in pool[:sample]:
|
|
73
|
+
truth = {wt.short(c) for c in wt.callers_of(s, name)}
|
|
74
|
+
got = callers_in.get(wt.short(key), set())
|
|
75
|
+
tp += len(truth & got)
|
|
76
|
+
fn += len(truth - got)
|
|
77
|
+
fp += len(got - truth)
|
|
78
|
+
truth_pairs |= {(c, wt.short(key)) for c in truth}
|
|
79
|
+
src_prec = {}
|
|
80
|
+
for src_name, pairs in by_src.items():
|
|
81
|
+
judged = [(a, b) for a, b in pairs if b.split("::")[-1] in unique and unique[b.split("::")[-1]] and wt.short(unique[b.split("::")[-1]]) == b]
|
|
82
|
+
if judged:
|
|
83
|
+
ok = sum(1 for a, b in judged if (a, b) in truth_pairs or a in {wt.short(c) for c in wt.callers_of(s, b.split("::")[-1])})
|
|
84
|
+
src_prec[src_name] = {"judged": len(judged), "precision": round(ok / len(judged), 3)}
|
|
85
|
+
rec, prec = tp / max(1, tp + fn), tp / max(1, tp + fp)
|
|
86
|
+
return {"sample": min(sample, len(pool)), "recall": round(rec, 3), "precision": round(prec, 3), "f1": round(f1(prec, rec), 3),
|
|
87
|
+
"edges_by_source": {k: len(v) for k, v in by_src.items()}, "precision_by_source": src_prec}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def inheritance_family(s, edges, rnd):
|
|
91
|
+
base = [(a, b) for a, b, p in edges["CALLS"] if p.get("source") == "ast-typed-receiver-base"]
|
|
92
|
+
ok = 0
|
|
93
|
+
for a, b in base:
|
|
94
|
+
key = (a["natural_key"].split("::")[0], a["natural_key"].split("::", 1)[1])
|
|
95
|
+
meth = b["natural_key"].split(".")[-1]
|
|
96
|
+
if any(called == meth for _kind, _base, called in s["calls"].get(key, ())):
|
|
97
|
+
ok += 1
|
|
98
|
+
sample = rnd.sample(base, min(8, len(base)))
|
|
99
|
+
return {"edges": len(base), "caller_calls_that_name": round(ok / max(1, len(base)), 3),
|
|
100
|
+
"examples": [f"{a['natural_key']} -> {b['natural_key']}" for a, b in sample]}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _test_calls(root: Path, test_key: str, cache: dict) -> set[str]:
|
|
104
|
+
"""Names called inside the test function at `path:line` (a TestCase natural key)."""
|
|
105
|
+
path, _, line = test_key.rpartition(":")
|
|
106
|
+
if path not in cache:
|
|
107
|
+
try:
|
|
108
|
+
cache[path] = ast.parse((root / path).read_text(errors="replace"))
|
|
109
|
+
except (OSError, SyntaxError, ValueError):
|
|
110
|
+
cache[path] = None
|
|
111
|
+
tree = cache[path]
|
|
112
|
+
if tree is None or not line.isdigit():
|
|
113
|
+
return set()
|
|
114
|
+
at = int(line)
|
|
115
|
+
# the innermost function enclosing the line: a marker edge points at where the test names the ticket
|
|
116
|
+
best = None
|
|
117
|
+
for n in ast.walk(tree):
|
|
118
|
+
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.lineno <= at <= getattr(n, "end_lineno", n.lineno):
|
|
119
|
+
if best is None or n.lineno > best.lineno:
|
|
120
|
+
best = n
|
|
121
|
+
if best is None:
|
|
122
|
+
return set()
|
|
123
|
+
return {c.func.id if isinstance(c.func, ast.Name) else c.func.attr for c in ast.walk(best)
|
|
124
|
+
if isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute))}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def tested_by_family(root, s, edges, rnd, sample=600):
|
|
128
|
+
defs_by_file = collections.defaultdict(set)
|
|
129
|
+
for name, keys in s["defs"].items():
|
|
130
|
+
for path, _q in keys:
|
|
131
|
+
defs_by_file[path].add(name)
|
|
132
|
+
by_src = collections.defaultdict(list)
|
|
133
|
+
for a, b, p in edges["TESTED_BY"]:
|
|
134
|
+
by_src[p.get("source", "?")].append((a, b))
|
|
135
|
+
cache: dict = {}
|
|
136
|
+
out = {}
|
|
137
|
+
for src_name, pairs in by_src.items():
|
|
138
|
+
pick = rnd.sample(pairs, min(sample, len(pairs)))
|
|
139
|
+
ok = 0
|
|
140
|
+
for a, b in pick:
|
|
141
|
+
called = _test_calls(root, b["natural_key"], cache)
|
|
142
|
+
if a["type"] == "Symbol":
|
|
143
|
+
target = {a["natural_key"].split(".")[-1].split("::")[-1]}
|
|
144
|
+
else:
|
|
145
|
+
target = defs_by_file.get(a["natural_key"], set())
|
|
146
|
+
ok += bool(called & target)
|
|
147
|
+
out[src_name] = {"edges": len(pairs), "judged": len(pick), "precision": round(ok / max(1, len(pick)), 3)}
|
|
148
|
+
return out
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def similar_family(nodes, edges, rnd):
|
|
152
|
+
parent, comp = {}, {}
|
|
153
|
+
for (t, k), n in nodes.items():
|
|
154
|
+
if t == "Ticket":
|
|
155
|
+
comp[k] = (n.get("properties") or {}).get("component")
|
|
156
|
+
for a, b, _p in edges["CHILD_OF"]:
|
|
157
|
+
parent[a["natural_key"]] = b["natural_key"]
|
|
158
|
+
|
|
159
|
+
def related(x, y):
|
|
160
|
+
return (parent.get(x) and parent.get(x) == parent.get(y)) or (comp.get(x) and comp.get(x) == comp.get(y))
|
|
161
|
+
|
|
162
|
+
kept = collections.defaultdict(list)
|
|
163
|
+
for a, b, p in edges["SIMILAR_TO"]:
|
|
164
|
+
kept[a["natural_key"]].append((float(p.get("weighted_score") or 0), b["natural_key"]))
|
|
165
|
+
top = [(x, y) for x, lst in kept.items() for _s, y in sorted(lst, reverse=True)[:8]]
|
|
166
|
+
tickets = [k for (t, k) in nodes if t == "Ticket"]
|
|
167
|
+
rand = [(rnd.choice(tickets), rnd.choice(tickets)) for _ in range(5000)]
|
|
168
|
+
return {"edges": len(edges["SIMILAR_TO"]), "tickets_with_neighbours": len(kept),
|
|
169
|
+
"edges_per_ticket": round(len(edges["SIMILAR_TO"]) / max(1, len(kept)), 1),
|
|
170
|
+
"precision_at_8": round(sum(1 for x, y in top if related(x, y)) / max(1, len(top)), 3),
|
|
171
|
+
"random_pair_rate": round(sum(1 for x, y in rand if x != y and related(x, y)) / len(rand), 3)}
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
_URL = re.compile(r"""(?:fetch|apiFetch|api|request|get|post|put|patch|del|delete)\s*(?:<[^>]*>)?\(\s*([`'"])(.*?)\1""", re.S)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def calls_api_family(root, nodes, edges):
|
|
178
|
+
routes = []
|
|
179
|
+
for (t, k), n in nodes.items():
|
|
180
|
+
if t == "Endpoint" and (n.get("properties") or {}).get("owner_symbol"):
|
|
181
|
+
path = (n["properties"].get("path") or k.split(" ", 1)[-1]).rstrip("/")
|
|
182
|
+
routes.append(re.compile("^" + re.sub(r"\\\{[^}]+\\\}", "[^/]+", re.escape(path)) + "$"))
|
|
183
|
+
files = subprocess.run(["git", "ls-files", "--", "*.ts", "*.tsx", "*.js", "*.jsx"], cwd=root, capture_output=True, text=True).stdout.split()
|
|
184
|
+
truth = set()
|
|
185
|
+
for rel in files:
|
|
186
|
+
if "node_modules" in rel or rel.endswith(".d.ts"):
|
|
187
|
+
continue
|
|
188
|
+
try:
|
|
189
|
+
text = (root / rel).read_text(errors="replace")
|
|
190
|
+
except OSError:
|
|
191
|
+
continue
|
|
192
|
+
for _q, url in _URL.findall(text):
|
|
193
|
+
path = re.sub(r"\$\{[^}]*\}", "x", url.split("?")[0])
|
|
194
|
+
path = re.sub(r"^\$\{[^}]*\}|^https?://[^/]+", "", path).rstrip("/")
|
|
195
|
+
if path.startswith("/") and any(r.match(path) for r in routes):
|
|
196
|
+
truth.add(rel)
|
|
197
|
+
got = {a["natural_key"].split("::")[0] for a, _b, _p in edges["CALLS_API"]}
|
|
198
|
+
return {"frontend_files_calling_a_route": len(truth), "graph_files": len(got & truth) if truth else len(got),
|
|
199
|
+
"recall": round(len(got & truth) / max(1, len(truth)), 3), "edges": len(edges["CALLS_API"])}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def raises_family(s, unique, edges):
|
|
203
|
+
truth = {(wt.short(k), msg) for k, msg in s["raises"] if unique.get(k[1].split(".")[-1]) == k}
|
|
204
|
+
got = {(short(a["natural_key"]), (b.get("properties") or {}).get("message_template")) for a, b, _p in edges["RAISES"]}
|
|
205
|
+
got_fn = {x for x, _ in got}
|
|
206
|
+
hit = sum(1 for fn, _msg in truth if fn in got_fn)
|
|
207
|
+
return {"truth": len(truth), "recall_by_function": round(hit / max(1, len(truth)), 3)}
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
_READ = re.compile(r"""os\.(?:environ(?:\.get|\.setdefault)?\s*[\[(]|getenv\s*\()\s*['"]([A-Z][A-Z0-9_]+)['"]""")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def reads_family(root, edges):
|
|
214
|
+
files = subprocess.run(["git", "ls-files", "--recurse-submodules", "--", "*.py"], cwd=root, capture_output=True, text=True).stdout.split()
|
|
215
|
+
truth = set()
|
|
216
|
+
for rel in files:
|
|
217
|
+
if wt.is_test_file(rel) or re.search(r"(^|/)tests?/", rel):
|
|
218
|
+
continue
|
|
219
|
+
try:
|
|
220
|
+
for name in _READ.findall((root / rel).read_text(errors="replace")):
|
|
221
|
+
truth.add((rel, name))
|
|
222
|
+
except OSError:
|
|
223
|
+
continue
|
|
224
|
+
got = {(a["natural_key"].split("::")[0], b["natural_key"]) for a, b, _p in edges["READS"]}
|
|
225
|
+
return {"truth": len(truth), "recall": round(len(truth & got) / max(1, len(truth)), 3), "graph": len(got)}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def coverage_family(s, unique, nodes, edges, rnd, sample=200):
|
|
229
|
+
callers_in = collections.defaultdict(set)
|
|
230
|
+
for a, b, _p in edges["CALLS"]:
|
|
231
|
+
callers_in[b["natural_key"]].add(short(a["natural_key"]))
|
|
232
|
+
pool = [(n, k) for n, k in unique.items() if not wt.is_test_file(k[0]) and wt.callers_of(s, n)]
|
|
233
|
+
rnd.shuffle(pool)
|
|
234
|
+
marked = honest = 0
|
|
235
|
+
for name, key in pool[:sample]:
|
|
236
|
+
node = nodes.get(("Symbol", f"{key[0]}::{key[1]}"))
|
|
237
|
+
if not node or (node.get("properties") or {}).get("unresolved_calls"):
|
|
238
|
+
continue
|
|
239
|
+
marked += 1
|
|
240
|
+
truth = {wt.short(c) for c in wt.callers_of(s, name)}
|
|
241
|
+
honest += truth <= callers_in.get(node["natural_key"], set())
|
|
242
|
+
return {"marked_complete": marked, "truly_complete": round(honest / max(1, marked), 3)}
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def main() -> int:
|
|
246
|
+
ap = argparse.ArgumentParser()
|
|
247
|
+
ap.add_argument("--dir", required=True)
|
|
248
|
+
ap.add_argument("--repo", required=True)
|
|
249
|
+
ap.add_argument("--json")
|
|
250
|
+
ap.add_argument("--seed", type=int, default=314)
|
|
251
|
+
ns = ap.parse_args()
|
|
252
|
+
root, d = Path(ns.repo), Path(ns.dir)
|
|
253
|
+
nodes, edges = load(d)
|
|
254
|
+
s = wt.scan(root)
|
|
255
|
+
unique = {n: ks[0] for n, ks in s["defs"].items() if len(ks) == 1 and not n.startswith("test_") and not n.startswith("__")}
|
|
256
|
+
out = {"graph": str(d), "repo": str(root),
|
|
257
|
+
"CALLS": calls_family(s, unique, edges, random.Random(ns.seed)),
|
|
258
|
+
"inheritance": inheritance_family(s, edges, random.Random(ns.seed)),
|
|
259
|
+
"TESTED_BY": tested_by_family(root, s, edges, random.Random(ns.seed)),
|
|
260
|
+
"SIMILAR_TO": similar_family(nodes, edges, random.Random(ns.seed)),
|
|
261
|
+
"CALLS_API": calls_api_family(root, nodes, edges),
|
|
262
|
+
"RAISES": raises_family(s, unique, edges),
|
|
263
|
+
"READS": reads_family(root, edges),
|
|
264
|
+
"coverage": coverage_family(s, unique, nodes, edges, random.Random(ns.seed))}
|
|
265
|
+
print(json.dumps(out, indent=1))
|
|
266
|
+
if ns.json:
|
|
267
|
+
Path(ns.json).write_text(json.dumps(out, indent=1))
|
|
268
|
+
return 0
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
if __name__ == "__main__":
|
|
272
|
+
sys.exit(main())
|