devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/build_symbols.py
ADDED
|
@@ -0,0 +1,1394 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""devkb Symbol-graph refresh (BL-3/BL-3b/BL-3c/BL-4b, body/legs model) --
|
|
3
|
+
standalone, NOT part of devkb/build.py's regular run.
|
|
4
|
+
|
|
5
|
+
Usage: python3 devkb/build_symbols.py --project CODE
|
|
6
|
+
|
|
7
|
+
Extracts function/class/method-level `Symbol` nodes from every git-tracked
|
|
8
|
+
Python AND TypeScript/TSX file -- the "as-built design" layer the body/legs
|
|
9
|
+
model calls for -- plus:
|
|
10
|
+
- BL-3b: cross-file CALLS resolution for Python (same-file first, then
|
|
11
|
+
via a file's own imports to another file's symbol) -- widened from the
|
|
12
|
+
original same-file-only scope specifically because this build is
|
|
13
|
+
already decoupled from devkb/build.py's hot path (the same reasoning
|
|
14
|
+
that justified the decoupling in the first place: pay real analysis
|
|
15
|
+
cost once, on an explicit refresh, not never).
|
|
16
|
+
- BL-3c: TypeScript/TSX Symbol extraction (devkb/ts_symbols.py) -- devkb
|
|
17
|
+
was Python-only until now, meaning 0 of any registered project's real
|
|
18
|
+
frontend code -- the actual user-facing behavior -- was in the graph.
|
|
19
|
+
- BL-4b: backend FastAPI route capture (ported from discovery/
|
|
20
|
+
extract_python.py's own _decorator_route/_router_prefix) plus a
|
|
21
|
+
cross-language CALLS_API edge resolving a frontend Symbol's own
|
|
22
|
+
get/post/put(...) call site to the real backend Symbol that handles
|
|
23
|
+
it -- the literal "UI action -> backend code" connective tissue a bug
|
|
24
|
+
trace / sequence analysis / user-journey audit actually needs.
|
|
25
|
+
|
|
26
|
+
Deliberately decoupled from the regular fast build: a standalone extractor with its own persisted
|
|
27
|
+
artifact, read by multiple consumers rather than re-derived by each one.
|
|
28
|
+
Writes `.devkb/symbols-{nodes,edges,manifest}.jsonl`. devkb/build.py's own
|
|
29
|
+
regular run reads these (if present) and merges them into the main store;
|
|
30
|
+
it never regenerates them. Re-run this script explicitly whenever the
|
|
31
|
+
Symbol graph needs to catch up with real source changes -- its own
|
|
32
|
+
manifest's git_head/built_at make that staleness checkable instead of
|
|
33
|
+
silent, the gap confirmed live in discovery/graph.json (no such fields at
|
|
34
|
+
all) and deliberately not repeated here.
|
|
35
|
+
"""
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import argparse
|
|
39
|
+
import ast
|
|
40
|
+
import re
|
|
41
|
+
import json
|
|
42
|
+
import sys
|
|
43
|
+
from datetime import datetime, timezone
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
|
|
46
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
47
|
+
|
|
48
|
+
from devkb import ts_symbols # noqa: E402
|
|
49
|
+
from devkb.extractors.tickets_git import _run # noqa: E402
|
|
50
|
+
from devkb.gitutil import git_head # noqa: E402
|
|
51
|
+
from devkb.model import DevKBGraph # noqa: E402
|
|
52
|
+
from devkb.project_context import ProjectContext, resolve # noqa: E402
|
|
53
|
+
|
|
54
|
+
# ── FastAPI route capture, ported from discovery/extract_python.py's own
|
|
55
|
+
# _decorator_route/_router_prefix (BL-4b) -- same regex/AST logic, not a
|
|
56
|
+
# reinvention, so a route this repo's own discovery/ already resolves
|
|
57
|
+
# resolves the same way here. ──────────────────────────────────────────────
|
|
58
|
+
_HTTP_METHODS = {"get", "post", "put", "delete", "patch"}
|
|
59
|
+
_ROUTER_NAMES = {"router", "app"}
|
|
60
|
+
# Routers are often named for what they serve (`public_router`, `user_router`, `admin_app`) or by framework habit
|
|
61
|
+
# (`api`, `bp`): the truth generator accepted any receiver and the graph accepted two names, so a whole router's
|
|
62
|
+
# handlers were invisible. The decorator must still carry a string path, which keeps `cache.get(...)` out.
|
|
63
|
+
_ROUTER_NAME = re.compile(r"(?:^|_)(?:router|app|api|bp|blueprint)$", re.IGNORECASE)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _odoo_route(dec: ast.expr) -> tuple[str, str] | None:
|
|
67
|
+
"""`@http.route('/path', methods=['POST'], ...)` (ODSF-304). Odoo's own
|
|
68
|
+
controller decorator, which the FastAPI-shaped matcher below never saw, so
|
|
69
|
+
one project's 143 routes were reported as 0. A list of paths or methods
|
|
70
|
+
yields the first of each: one handler is one Endpoint here, and the extra
|
|
71
|
+
aliases are the same code."""
|
|
72
|
+
if not (isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute)
|
|
73
|
+
and dec.func.attr == "route" and isinstance(dec.func.value, ast.Name)
|
|
74
|
+
and dec.func.value.id == "http"):
|
|
75
|
+
return None
|
|
76
|
+
first: ast.expr | None = dec.args[0] if dec.args else next(
|
|
77
|
+
(kw.value for kw in dec.keywords if kw.arg == "route"), None)
|
|
78
|
+
if isinstance(first, (ast.List, ast.Tuple)) and first.elts:
|
|
79
|
+
first = first.elts[0]
|
|
80
|
+
if not (isinstance(first, ast.Constant) and isinstance(first.value, str)):
|
|
81
|
+
return None
|
|
82
|
+
method = "ANY"
|
|
83
|
+
for kw in dec.keywords:
|
|
84
|
+
if kw.arg == "methods" and isinstance(kw.value, (ast.List, ast.Tuple)) and kw.value.elts:
|
|
85
|
+
head = kw.value.elts[0]
|
|
86
|
+
if isinstance(head, ast.Constant) and isinstance(head.value, str):
|
|
87
|
+
method = head.value.upper()
|
|
88
|
+
return method, first.value
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _decorator_route(dec: ast.expr) -> tuple[str, str] | None:
|
|
92
|
+
odoo = _odoo_route(dec)
|
|
93
|
+
if odoo is not None:
|
|
94
|
+
return odoo
|
|
95
|
+
if not isinstance(dec, ast.Call) or not isinstance(dec.func, ast.Attribute):
|
|
96
|
+
return None
|
|
97
|
+
if dec.func.attr not in _HTTP_METHODS:
|
|
98
|
+
return None
|
|
99
|
+
if not isinstance(dec.func.value, ast.Name) or not (dec.func.value.id in _ROUTER_NAMES or _ROUTER_NAME.search(dec.func.value.id)):
|
|
100
|
+
return None
|
|
101
|
+
if not dec.args or not isinstance(dec.args[0], ast.Constant) or not isinstance(dec.args[0].value, str):
|
|
102
|
+
return None
|
|
103
|
+
return dec.func.attr.upper(), dec.args[0].value
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _router_prefix(tree: ast.Module) -> str:
|
|
107
|
+
"""The prefix a module's router declares. Any constructor named like a router counts (`APIRouter`,
|
|
108
|
+
`fastapi.APIRouter`, a project's own `PrefectRouter`), and Flask's `Blueprint(url_prefix=...)`: reading only
|
|
109
|
+
`APIRouter` left a project's prefixes empty, so `POST /` in five modules collapsed into one Endpoint."""
|
|
110
|
+
for node in ast.walk(tree):
|
|
111
|
+
if not isinstance(node, ast.Call):
|
|
112
|
+
continue
|
|
113
|
+
name = node.func.id if isinstance(node.func, ast.Name) else node.func.attr if isinstance(node.func, ast.Attribute) else ""
|
|
114
|
+
if not (name.endswith("Router") or name == "Blueprint"):
|
|
115
|
+
continue
|
|
116
|
+
for kw in node.keywords:
|
|
117
|
+
if kw.arg in ("prefix", "url_prefix") and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
|
|
118
|
+
return kw.value.value
|
|
119
|
+
return ""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _list_python_files(repo_root: Path) -> list[str]:
|
|
123
|
+
# git ls-files (not rglob) so this automatically respects .gitignore --
|
|
124
|
+
# no manual __pycache__/venv exclusion list to keep in sync by hand.
|
|
125
|
+
out = _run(["git", "ls-files", "--recurse-submodules", "--", "*.py"], repo_root)
|
|
126
|
+
return [f for f in out.splitlines() if f.strip()]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# Third-party and generated JavaScript is not the project's own behaviour.
|
|
130
|
+
_JS_SKIP = re.compile(
|
|
131
|
+
r"(\.min\.js$|\.test\.[cm]?js$|\.spec\.[cm]?js$|(^|/)(vendor|node_modules|output|dist|build)/)")
|
|
132
|
+
# ODSF-314: `lib/` holds vendored code only as compiled JavaScript; a project's own TypeScript lives there too
|
|
133
|
+
# (OCO's webapp/frontend/src/lib/api.ts is its whole API layer), so `lib/` is skipped for .js/.mjs only.
|
|
134
|
+
_JS_LIB = re.compile(r"(^|/)(lib|libs)/[^/].*\.m?js$")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _list_ts_files(repo_root: Path) -> list[str]:
|
|
138
|
+
out = _run(["git", "ls-files", "--recurse-submodules", "--", "*.ts", "*.tsx", "*.js", "*.mjs"], repo_root)
|
|
139
|
+
return [
|
|
140
|
+
f for f in out.splitlines()
|
|
141
|
+
if f.strip() and not f.endswith(".d.ts") and not _JS_SKIP.search(f) and not _JS_LIB.search(f)
|
|
142
|
+
]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ODSF-304: an Odoo addon's behaviour lives in XML as much as in Python, and
|
|
146
|
+
# none of it had a node. Only the declarations a person uses or navigates by.
|
|
147
|
+
_XML_TAG = re.compile(r"<(template|record|menuitem|t)\b([^>]*)>", re.IGNORECASE)
|
|
148
|
+
_XML_ATTR = re.compile(r"""([\w:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')""")
|
|
149
|
+
_XML_RECORD_KINDS = {"ir.ui.view": "view", "ir.cron": "cron", "res.groups": "group"}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _odoo_xml_symbols(repo_root: Path) -> dict[str, list[dict]]:
|
|
153
|
+
"""{file: [{qualified_name, kind, line_start}, ...]} for QWeb templates,
|
|
154
|
+
views, actions, menus, cron jobs and security groups declared in tracked
|
|
155
|
+
XML. Regex over the tag, not a full parse: Odoo data files carry
|
|
156
|
+
`t-` attributes and entities that make a strict parser reject real files."""
|
|
157
|
+
out: dict[str, list[dict]] = {}
|
|
158
|
+
for rel in _run(["git", "ls-files", "--recurse-submodules", "--", "*.xml"], repo_root).splitlines():
|
|
159
|
+
if not rel.strip():
|
|
160
|
+
continue
|
|
161
|
+
try:
|
|
162
|
+
text = (repo_root / rel).read_text(errors="replace")
|
|
163
|
+
except OSError:
|
|
164
|
+
continue
|
|
165
|
+
found: list[dict] = []
|
|
166
|
+
for match in _XML_TAG.finditer(text):
|
|
167
|
+
tag = match.group(1).lower()
|
|
168
|
+
attrs = {m.group(1): (m.group(2) if m.group(2) is not None else m.group(3))
|
|
169
|
+
for m in _XML_ATTR.finditer(match.group(2))}
|
|
170
|
+
line = text.count("\n", 0, match.start()) + 1
|
|
171
|
+
if tag == "template" and attrs.get("id"):
|
|
172
|
+
found.append({"qualified_name": attrs["id"], "kind": "qweb_template", "line_start": line})
|
|
173
|
+
elif tag == "menuitem" and attrs.get("id"):
|
|
174
|
+
found.append({"qualified_name": attrs["id"], "kind": "menu", "line_start": line})
|
|
175
|
+
elif tag == "t" and attrs.get("t-name"):
|
|
176
|
+
found.append({"qualified_name": attrs["t-name"], "kind": "qweb_template", "line_start": line})
|
|
177
|
+
elif tag == "record" and attrs.get("id") and attrs.get("model"):
|
|
178
|
+
model_name = attrs["model"]
|
|
179
|
+
kind = _XML_RECORD_KINDS.get(model_name) or (
|
|
180
|
+
"action" if model_name.startswith("ir.actions.") else None)
|
|
181
|
+
if kind:
|
|
182
|
+
found.append({"qualified_name": attrs["id"], "kind": kind, "line_start": line})
|
|
183
|
+
if found:
|
|
184
|
+
out[rel] = found
|
|
185
|
+
return out
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _first_line(docstring: str | None) -> str | None:
|
|
189
|
+
if not docstring:
|
|
190
|
+
return None
|
|
191
|
+
return docstring.strip().splitlines()[0].strip()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
EXPR = "<expr>"
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _call_name(func: ast.expr) -> tuple[str | None, str] | None:
|
|
198
|
+
"""(receiver_name_or_None, short_name) for a call target. The receiver
|
|
199
|
+
is used for BL-3b's cross-file resolution (module.func() -> follow
|
|
200
|
+
`module`'s own import to the right file); same-file/global fallback
|
|
201
|
+
still works from the short name alone when there's no usable receiver
|
|
202
|
+
(a bare `func()`, or a call on something more complex than a plain
|
|
203
|
+
name, e.g. `get_thing().run()`)."""
|
|
204
|
+
if isinstance(func, ast.Name):
|
|
205
|
+
return None, func.id
|
|
206
|
+
if isinstance(func, ast.Attribute):
|
|
207
|
+
# A call on an expression (`Path(p).resolve()`, `super().setUp()`) has a receiver, just not a name: EXPR,
|
|
208
|
+
# never None, so it is not mistaken for a bare call to a same-named function of this file.
|
|
209
|
+
receiver = func.value.id if isinstance(func.value, ast.Name) else _dotted(func.value) or EXPR
|
|
210
|
+
return receiver, func.attr
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _dotted(node: ast.expr) -> str | None:
|
|
215
|
+
"""`a.b.c` for an attribute chain rooted at a plain name (`prefect.context` in `prefect.context.get_run_context()`,
|
|
216
|
+
after `import prefect.context`), else None. Resolved only when its root is an imported module; any other chain
|
|
217
|
+
(`self._store`, `request.app`) is treated as an expression, as before."""
|
|
218
|
+
parts: list[str] = []
|
|
219
|
+
while isinstance(node, ast.Attribute):
|
|
220
|
+
parts.insert(0, node.attr)
|
|
221
|
+
node = node.value
|
|
222
|
+
if isinstance(node, ast.Name) and parts and node.id not in ("self", "cls"):
|
|
223
|
+
return ".".join([node.id, *parts])
|
|
224
|
+
return None
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _calls_in_body(node: ast.AST) -> list[tuple[str | None, str]]:
|
|
228
|
+
"""(receiver, short_name) calls made directly within node's own body --
|
|
229
|
+
explicitly NOT descending into a nested function/class def's body, so a
|
|
230
|
+
nested def's own calls get attributed to ITS symbol when the visitor
|
|
231
|
+
reaches it, not double-counted here."""
|
|
232
|
+
calls: list[tuple[str | None, str]] = []
|
|
233
|
+
|
|
234
|
+
def walk(n: ast.AST, top: bool) -> None:
|
|
235
|
+
for child in ast.iter_child_nodes(n):
|
|
236
|
+
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and not top:
|
|
237
|
+
continue
|
|
238
|
+
if isinstance(child, ast.Call):
|
|
239
|
+
hit = _call_name(child.func)
|
|
240
|
+
if hit:
|
|
241
|
+
calls.append(hit)
|
|
242
|
+
walk(child, top=False)
|
|
243
|
+
|
|
244
|
+
walk(node, top=True)
|
|
245
|
+
return calls
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
# A call made when the module is imported (`app_dirs = get_app_dirs()`), or in a class body, outside every function.
|
|
249
|
+
# It has no Symbol to start a CALLS edge from (Symbol is a function, class or method), so it is resolved like any call
|
|
250
|
+
# and recorded on the called Symbol as `module_callers`: the files a signature change breaks.
|
|
251
|
+
MODULE_CALLER = "<module>"
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _module_level_calls(tree: ast.Module) -> list[tuple[str | None, str]]:
|
|
255
|
+
calls: list[tuple[str | None, str]] = []
|
|
256
|
+
|
|
257
|
+
def walk(n: ast.AST) -> None:
|
|
258
|
+
for child in ast.iter_child_nodes(n):
|
|
259
|
+
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
|
260
|
+
continue
|
|
261
|
+
if isinstance(child, ast.Call):
|
|
262
|
+
hit = _call_name(child.func)
|
|
263
|
+
if hit:
|
|
264
|
+
calls.append(hit)
|
|
265
|
+
walk(child)
|
|
266
|
+
|
|
267
|
+
walk(tree)
|
|
268
|
+
return calls
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _imports(tree: ast.AST) -> list[tuple[str, str | None]]:
|
|
272
|
+
"""[(local_name, dotted_module_or_None), ...] -- dotted_module is None
|
|
273
|
+
for a bare `import X` (X itself is the module, not a from-import of a
|
|
274
|
+
specific name from it). Shared with extractors/symbol_tests.py (BL-4),
|
|
275
|
+
which does the same "resolve an imported name to a Symbol" job for
|
|
276
|
+
test files -- one implementation, not two copies to keep in sync."""
|
|
277
|
+
out: list[tuple[str, str | None]] = []
|
|
278
|
+
for node in ast.walk(tree):
|
|
279
|
+
if isinstance(node, ast.ImportFrom) and node.module:
|
|
280
|
+
for alias in node.names:
|
|
281
|
+
out.append((alias.asname or alias.name, node.module))
|
|
282
|
+
elif isinstance(node, ast.Import):
|
|
283
|
+
for alias in node.names:
|
|
284
|
+
out.append((alias.asname or alias.name.split(".")[0], None))
|
|
285
|
+
return out
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _module_paths(dotted: str) -> list[str]:
|
|
289
|
+
slash = dotted.replace(".", "/")
|
|
290
|
+
return [f"{slash}.py", f"{slash}/__init__.py"]
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _import_descriptors(tree: ast.AST) -> list[tuple[int, str, str | None, str]]:
|
|
294
|
+
"""[(level, module, name_or_None, local_name), ...] -- every import with
|
|
295
|
+
what `_imports` throws away (ODSF-305): the module an alias stands for,
|
|
296
|
+
the imported name, and the relative level. `import a.b as m` is
|
|
297
|
+
(0, "a.b", None, "m"); `from a import b as c` is (0, "a", "b", "c");
|
|
298
|
+
`from . import x` is (1, "", "x", "x")."""
|
|
299
|
+
out: list[tuple[int, str, str | None, str]] = []
|
|
300
|
+
for node in ast.walk(tree):
|
|
301
|
+
if isinstance(node, ast.ImportFrom):
|
|
302
|
+
for alias in node.names:
|
|
303
|
+
if alias.name != "*":
|
|
304
|
+
out.append((node.level or 0, node.module or "", alias.name, alias.asname or alias.name))
|
|
305
|
+
elif isinstance(node, ast.Import):
|
|
306
|
+
for alias in node.names:
|
|
307
|
+
if alias.asname:
|
|
308
|
+
out.append((0, alias.name, None, alias.asname))
|
|
309
|
+
else:
|
|
310
|
+
top = alias.name.split(".")[0]
|
|
311
|
+
out.append((0, top, None, top))
|
|
312
|
+
return out
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _import_targets(
|
|
316
|
+
descriptors: list[tuple[int, str, str | None, str]], rel_path: str,
|
|
317
|
+
) -> tuple[dict[str, list[str]], dict[str, tuple[str, str]]]:
|
|
318
|
+
"""(receiver_modules, bare_symbols) for one file (ODSF-305).
|
|
319
|
+
|
|
320
|
+
receiver_modules[local] lists the dotted modules `local` may be, so a call
|
|
321
|
+
`local.func()` is looked up in each. `from pkg import mod` makes `mod` a
|
|
322
|
+
module `pkg.mod` -- the case the old resolver got wrong by searching
|
|
323
|
+
`pkg` itself. bare_symbols[local] = (module, original_name) for a bare
|
|
324
|
+
call `local()` made through `from module import original_name as local`.
|
|
325
|
+
Relative imports are anchored on the caller's own directory."""
|
|
326
|
+
here = rel_path.split("/")[:-1]
|
|
327
|
+
receivers: dict[str, list[str]] = {}
|
|
328
|
+
bare: dict[str, tuple[str, str]] = {}
|
|
329
|
+
for level, module, name, local in descriptors:
|
|
330
|
+
if level:
|
|
331
|
+
keep = here[: len(here) - (level - 1)] if level - 1 <= len(here) else []
|
|
332
|
+
base = ".".join(keep + ([module] if module else []))
|
|
333
|
+
else:
|
|
334
|
+
base = module
|
|
335
|
+
if not base and name is None:
|
|
336
|
+
continue
|
|
337
|
+
if name is None:
|
|
338
|
+
receivers.setdefault(local, []).append(base)
|
|
339
|
+
else:
|
|
340
|
+
receivers.setdefault(local, []).append(f"{base}.{name}" if base else name)
|
|
341
|
+
if base:
|
|
342
|
+
bare[local] = (base, name)
|
|
343
|
+
return receivers, bare
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _py_literals(tree: ast.AST) -> list[str]:
|
|
347
|
+
"""String constants that look like a Python file path in this file (ODSF-305):
|
|
348
|
+
`.../dev-orchestrator.py`, the way a test names a module it loads with
|
|
349
|
+
importlib because the file name is not a valid module name."""
|
|
350
|
+
return sorted({
|
|
351
|
+
n.value for n in ast.walk(tree)
|
|
352
|
+
if isinstance(n, ast.Constant) and isinstance(n.value, str)
|
|
353
|
+
and n.value.endswith(".py") and len(n.value) < 200 and "\n" not in n.value
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
_SUFFIX_CACHE: dict[int, tuple[int, dict[str, list[str]]]] = {}
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _suffix_index(per_file_symbols: dict[str, list[dict]]) -> dict[str, list[str]]:
|
|
361
|
+
"""Every trailing path of every file -> the files that end that way
|
|
362
|
+
(`a/b/c.py` is indexed as `c.py`, `b/c.py` and `a/b/c.py`). Built once per
|
|
363
|
+
symbol table so module lookup stays a dict hit on a repo with thousands of
|
|
364
|
+
files (ODSF-305)."""
|
|
365
|
+
cached = _SUFFIX_CACHE.get(id(per_file_symbols))
|
|
366
|
+
if cached and cached[0] == len(per_file_symbols):
|
|
367
|
+
return cached[1]
|
|
368
|
+
index: dict[str, list[str]] = {}
|
|
369
|
+
for path in per_file_symbols:
|
|
370
|
+
parts = path.split("/")
|
|
371
|
+
for i in range(len(parts)):
|
|
372
|
+
index.setdefault("/".join(parts[i:]), []).append(path)
|
|
373
|
+
_SUFFIX_CACHE[id(per_file_symbols)] = (len(per_file_symbols), index)
|
|
374
|
+
return index
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _resolve_via_literals(
|
|
378
|
+
literals: list[str], short_name: str, per_file_symbols: dict[str, list[dict]],
|
|
379
|
+
) -> str | None:
|
|
380
|
+
"""`short_name` in the one file this caller names by path, when exactly one
|
|
381
|
+
symbol matches across every file those path strings point at."""
|
|
382
|
+
index = _suffix_index(per_file_symbols)
|
|
383
|
+
files = {p for lit in literals for p in index.get(lit.lstrip("./"), [])}
|
|
384
|
+
hits = [f"{f}::{s['qualified_name']}" for f in sorted(files) for s in per_file_symbols[f]
|
|
385
|
+
if s["qualified_name"].rsplit(".", 1)[-1] == short_name]
|
|
386
|
+
return hits[0] if len(hits) == 1 else None
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# Re-exports: file -> {name it makes importable: (dotted module it imported that name from, original name)}, so
|
|
390
|
+
# `from pkg import f` resolves when `pkg/__init__.py` holds `from .impl import f`. Registered per symbol table (the
|
|
391
|
+
# table object itself is kept, so a reused id() can never pick up another build's map).
|
|
392
|
+
_REEXPORTS: dict[int, tuple[dict, dict[str, dict[str, tuple[str, str]]]]] = {}
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def register_reexports(per_file_symbols: dict[str, list[dict]], reexports: dict[str, dict[str, tuple[str, str]]]) -> None:
|
|
396
|
+
_REEXPORTS[id(per_file_symbols)] = (per_file_symbols, reexports)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def collect_reexports(repo_root: Path, rel_paths: list[str]) -> dict[str, dict[str, tuple[str, str]]]:
|
|
400
|
+
"""Every file's from-imports, as `_import_targets` resolves them (relative imports anchored on the file)."""
|
|
401
|
+
out: dict[str, dict[str, tuple[str, str]]] = {}
|
|
402
|
+
for rel in rel_paths:
|
|
403
|
+
try:
|
|
404
|
+
tree = ast.parse((repo_root / rel).read_text(errors="replace"))
|
|
405
|
+
except (SyntaxError, OSError, ValueError, RecursionError):
|
|
406
|
+
continue
|
|
407
|
+
bare = _import_targets(_import_descriptors(tree), rel)[1]
|
|
408
|
+
if bare:
|
|
409
|
+
out[rel] = bare
|
|
410
|
+
return out
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _resolve_cross_file_symbol(
|
|
414
|
+
dotted_module: str, caller_rel_path: str, short_name: str, per_file_symbols: dict[str, list[dict]],
|
|
415
|
+
_depth: int = 0,
|
|
416
|
+
) -> str | None:
|
|
417
|
+
"""Resolves `dotted_module`'s `short_name` symbol to a real
|
|
418
|
+
`path::qualified_name`, repo-root-relative first, then relative to
|
|
419
|
+
each ancestor directory of the caller's own file, closest first --
|
|
420
|
+
real per-service import convention this codebase actually uses
|
|
421
|
+
(`from app.middleware.auth import AuthMiddleware` inside
|
|
422
|
+
webapp/backend/ means webapp/backend/app/middleware/auth.py, not a
|
|
423
|
+
repo-root-level app/; confirmed live, not a hypothetical). None when
|
|
424
|
+
no candidate resolves to exactly one match -- skipped, not guessed."""
|
|
425
|
+
entry = _REEXPORTS.get(id(per_file_symbols))
|
|
426
|
+
reexports = entry[1] if entry and entry[0] is per_file_symbols else {}
|
|
427
|
+
|
|
428
|
+
def _match(guess: str) -> str | None:
|
|
429
|
+
hits = [s["qualified_name"] for s in per_file_symbols.get(guess) or []
|
|
430
|
+
if s["qualified_name"].rsplit(".", 1)[-1] == short_name]
|
|
431
|
+
if len(hits) == 1:
|
|
432
|
+
return f"{guess}::{hits[0]}"
|
|
433
|
+
if hits:
|
|
434
|
+
return None
|
|
435
|
+
# Not defined here: the module may re-export it (`from .wallets import get_wallet` in a package's
|
|
436
|
+
# `__init__.py`). Follow that import, a bounded number of hops.
|
|
437
|
+
via = reexports.get(guess, {}).get(short_name)
|
|
438
|
+
if via and _depth < 6:
|
|
439
|
+
return _resolve_cross_file_symbol(via[0], guess, via[1], per_file_symbols, _depth + 1)
|
|
440
|
+
return None
|
|
441
|
+
|
|
442
|
+
candidates = _module_paths(dotted_module)
|
|
443
|
+
for guess in candidates:
|
|
444
|
+
hit = _match(guess)
|
|
445
|
+
if hit:
|
|
446
|
+
return hit
|
|
447
|
+
|
|
448
|
+
parts = caller_rel_path.split("/")[:-1] # caller's own containing directories, deepest last
|
|
449
|
+
for i in range(len(parts), 0, -1):
|
|
450
|
+
prefix = "/".join(parts[:i])
|
|
451
|
+
for guess in candidates:
|
|
452
|
+
hit = _match(f"{prefix}/{guess}")
|
|
453
|
+
if hit:
|
|
454
|
+
return hit
|
|
455
|
+
|
|
456
|
+
# ODSF-305: a module reached through a `sys.path.insert(...)` lives in a
|
|
457
|
+
# directory no ancestor rule can guess (`import sdp_landing` -> sdp/orchestrator/).
|
|
458
|
+
# Accept it only when exactly one tracked file ends with the module's path:
|
|
459
|
+
# a unique MODULE name, never a unique symbol name, which was rejected above
|
|
460
|
+
# because `get`/`run`/`main` resolve to the wrong file.
|
|
461
|
+
index = _suffix_index(per_file_symbols)
|
|
462
|
+
found = {p for guess in candidates for p in index.get(guess, [])}
|
|
463
|
+
if len(found) == 1:
|
|
464
|
+
return _match(next(iter(found)))
|
|
465
|
+
return None
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
# ── ODSF-312: calls through an object whose class is known ──────────────────
|
|
469
|
+
# `self._store.add_edge(...)` and `store = RuntimeStore(); store.add_edge(...)` carry no module name, so the import
|
|
470
|
+
# resolver above never sees them; in OCO they were most of the calls into service classes. A receiver's class is
|
|
471
|
+
# taken only from what the code states: a parameter annotation (`store: RuntimeStore`, `Optional[...]`, `X | None`,
|
|
472
|
+
# a string annotation), a constructor call (`x = RuntimeStore(...)`, `with C(...) as x`, `x: C = ...`), a class-level
|
|
473
|
+
# annotation, or a `self.attr` set in any method of the class from one of those. Nothing is inferred from a return
|
|
474
|
+
# type or a factory, so a miss stays a miss rather than becoming a wrong edge.
|
|
475
|
+
def _annotation_class(ann: ast.expr | None) -> str | None:
|
|
476
|
+
if ann is None:
|
|
477
|
+
return None
|
|
478
|
+
if isinstance(ann, ast.Constant) and isinstance(ann.value, str):
|
|
479
|
+
try:
|
|
480
|
+
ann = ast.parse(ann.value, mode="eval").body
|
|
481
|
+
except SyntaxError:
|
|
482
|
+
return None
|
|
483
|
+
if isinstance(ann, ast.BinOp) and isinstance(ann.op, ast.BitOr): # X | None
|
|
484
|
+
sides = [s for s in (ann.left, ann.right) if not (isinstance(s, ast.Constant) and s.value is None)]
|
|
485
|
+
return _annotation_class(sides[0]) if len(sides) == 1 else None
|
|
486
|
+
if isinstance(ann, ast.Subscript) and isinstance(ann.value, (ast.Name, ast.Attribute)):
|
|
487
|
+
outer = ann.value.id if isinstance(ann.value, ast.Name) else ann.value.attr
|
|
488
|
+
return _annotation_class(ann.slice) if outer == "Optional" else None
|
|
489
|
+
return _class_ref(ann)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _class_ref(node: ast.expr) -> str | None:
|
|
493
|
+
"""`Cls` or `module.Cls` when the name looks like a class (capitalised), else None."""
|
|
494
|
+
if isinstance(node, ast.Name) and node.id[:1].isupper():
|
|
495
|
+
return node.id
|
|
496
|
+
if isinstance(node, ast.Attribute) and node.attr[:1].isupper() and isinstance(node.value, ast.Name):
|
|
497
|
+
return f"{node.value.id}.{node.attr}"
|
|
498
|
+
return None
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _constructed(value: ast.expr | None) -> str | None:
|
|
502
|
+
return _class_ref(value.func) if isinstance(value, ast.Call) else None
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _own_body(node: ast.AST):
|
|
506
|
+
"""Nodes of a function's own body, not descending into nested defs or classes (as _calls_in_body)."""
|
|
507
|
+
stack = list(ast.iter_child_nodes(node))
|
|
508
|
+
while stack:
|
|
509
|
+
n = stack.pop()
|
|
510
|
+
yield n
|
|
511
|
+
if not isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)):
|
|
512
|
+
stack.extend(ast.iter_child_nodes(n))
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _typed_calls(tree: ast.Module) -> list[tuple[str, str, str]]:
|
|
516
|
+
"""[(caller_qualname, class_ref, method)] for calls on a receiver whose class the code states."""
|
|
517
|
+
out: list[tuple[str, str, str]] = []
|
|
518
|
+
|
|
519
|
+
def params(fn) -> dict[str, str]:
|
|
520
|
+
a = fn.args
|
|
521
|
+
env = {}
|
|
522
|
+
for p in [*a.posonlyargs, *a.args, *a.kwonlyargs]:
|
|
523
|
+
cls = _annotation_class(p.annotation)
|
|
524
|
+
if cls:
|
|
525
|
+
env[p.arg] = cls
|
|
526
|
+
return env
|
|
527
|
+
|
|
528
|
+
def attr_types(cls_node: ast.ClassDef) -> dict[str, str]:
|
|
529
|
+
types: dict[str, str] = {}
|
|
530
|
+
for stmt in cls_node.body:
|
|
531
|
+
if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
|
|
532
|
+
cls = _annotation_class(stmt.annotation) or _constructed(stmt.value)
|
|
533
|
+
if cls:
|
|
534
|
+
types[stmt.target.id] = cls
|
|
535
|
+
for fn in cls_node.body:
|
|
536
|
+
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
537
|
+
continue
|
|
538
|
+
env = params(fn)
|
|
539
|
+
for n in _own_body(fn):
|
|
540
|
+
targets, value, ann = [], None, None
|
|
541
|
+
if isinstance(n, ast.Assign):
|
|
542
|
+
targets, value = n.targets, n.value
|
|
543
|
+
elif isinstance(n, ast.AnnAssign):
|
|
544
|
+
targets, value, ann = [n.target], n.value, n.annotation
|
|
545
|
+
for t in targets:
|
|
546
|
+
if isinstance(t, ast.Attribute) and isinstance(t.value, ast.Name) and t.value.id == "self":
|
|
547
|
+
cls = (_annotation_class(ann) or _constructed(value)
|
|
548
|
+
or (env.get(value.id) if isinstance(value, ast.Name) else None))
|
|
549
|
+
if cls:
|
|
550
|
+
types.setdefault(t.attr, cls)
|
|
551
|
+
return types
|
|
552
|
+
|
|
553
|
+
def visit_function(fn, qual: str, self_types: dict[str, str], enclosing: str = "") -> None:
|
|
554
|
+
env = params(fn)
|
|
555
|
+
for n in _own_body(fn):
|
|
556
|
+
if isinstance(n, ast.Assign) and len(n.targets) == 1 and isinstance(n.targets[0], ast.Name):
|
|
557
|
+
cls = _constructed(n.value)
|
|
558
|
+
if cls:
|
|
559
|
+
env[n.targets[0].id] = cls
|
|
560
|
+
elif isinstance(n, ast.AnnAssign) and isinstance(n.target, ast.Name):
|
|
561
|
+
cls = _annotation_class(n.annotation) or _constructed(n.value)
|
|
562
|
+
if cls:
|
|
563
|
+
env[n.target.id] = cls
|
|
564
|
+
elif isinstance(n, (ast.With, ast.AsyncWith)):
|
|
565
|
+
for item in n.items:
|
|
566
|
+
cls = _constructed(item.context_expr)
|
|
567
|
+
if cls and isinstance(item.optional_vars, ast.Name):
|
|
568
|
+
env[item.optional_vars.id] = cls
|
|
569
|
+
for n in _own_body(fn):
|
|
570
|
+
if not (isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)):
|
|
571
|
+
continue
|
|
572
|
+
recv, method = n.func.value, n.func.attr
|
|
573
|
+
cls = None
|
|
574
|
+
if isinstance(recv, ast.Name) and recv.id in ("self", "cls") and enclosing:
|
|
575
|
+
cls = "self:" + enclosing # ODSF-313: resolved on the enclosing class, then its bases
|
|
576
|
+
elif isinstance(recv, ast.Name):
|
|
577
|
+
cls = env.get(recv.id) or (recv.id if recv.id[:1].isupper() else None) # `Cls.static()` too
|
|
578
|
+
elif isinstance(recv, ast.Attribute) and isinstance(recv.value, ast.Name) and recv.value.id == "self":
|
|
579
|
+
cls = self_types.get(recv.attr)
|
|
580
|
+
elif isinstance(recv, ast.Call):
|
|
581
|
+
cls = _constructed(recv) # `RuntimeStore().add_edge(...)`, built and called in one expression
|
|
582
|
+
if cls:
|
|
583
|
+
out.append((qual, cls, method))
|
|
584
|
+
|
|
585
|
+
def walk(node, stack: list[str], self_types: dict[str, str], cls_qual: str = "") -> None:
|
|
586
|
+
for child in ast.iter_child_nodes(node):
|
|
587
|
+
if isinstance(child, ast.ClassDef):
|
|
588
|
+
walk(child, stack + [child.name], attr_types(child), ".".join(stack + [child.name]))
|
|
589
|
+
elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
590
|
+
qual = ".".join(stack + [child.name])
|
|
591
|
+
visit_function(child, qual, self_types, cls_qual)
|
|
592
|
+
walk(child, stack + [child.name], {}, "")
|
|
593
|
+
else:
|
|
594
|
+
walk(child, stack, self_types, cls_qual)
|
|
595
|
+
|
|
596
|
+
walk(tree, [], {})
|
|
597
|
+
return out
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _resolve_class(
|
|
601
|
+
cls_ref: str, rel_path: str, per_file_symbols: dict[str, list[dict]],
|
|
602
|
+
targets: tuple[dict[str, list[str]], dict[str, tuple[str, str]]],
|
|
603
|
+
) -> str | None:
|
|
604
|
+
"""`path::Cls` for a class named in `rel_path`: its own file, `from m import Cls`, or `m.Cls` with `import m`."""
|
|
605
|
+
if cls_ref.startswith("self:"):
|
|
606
|
+
return f"{rel_path}::{cls_ref[5:]}"
|
|
607
|
+
receivers, bare = targets
|
|
608
|
+
found: str | None = None
|
|
609
|
+
if "." not in cls_ref and any(s["qualified_name"] == cls_ref for s in per_file_symbols.get(rel_path, [])):
|
|
610
|
+
found = f"{rel_path}::{cls_ref}"
|
|
611
|
+
elif "." not in cls_ref and cls_ref in bare:
|
|
612
|
+
module, original = bare[cls_ref]
|
|
613
|
+
found = _resolve_cross_file_symbol(module, rel_path, original, per_file_symbols)
|
|
614
|
+
elif "." in cls_ref:
|
|
615
|
+
alias, name = cls_ref.split(".", 1)
|
|
616
|
+
for module in receivers.get(alias, []):
|
|
617
|
+
found = _resolve_cross_file_symbol(module, rel_path, name, per_file_symbols)
|
|
618
|
+
if found:
|
|
619
|
+
break
|
|
620
|
+
return found
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _resolve_typed_call(
|
|
624
|
+
cls_ref: str, method: str, rel_path: str, per_file_symbols: dict[str, list[dict]],
|
|
625
|
+
targets: tuple[dict[str, list[str]], dict[str, tuple[str, str]]],
|
|
626
|
+
bases: dict[str, list[str]] | None = None,
|
|
627
|
+
) -> str | None:
|
|
628
|
+
"""`path::Cls.method` for a method called on an instance of `cls_ref`, or None. The method must be defined on the
|
|
629
|
+
class or, through EXTENDS (ADR-SDP-DEVKB-006), on one of its bases, nearest first."""
|
|
630
|
+
found = _resolve_class(cls_ref, rel_path, per_file_symbols, targets)
|
|
631
|
+
seen: set[str] = set()
|
|
632
|
+
queue = [found] if found else []
|
|
633
|
+
while queue:
|
|
634
|
+
cls_key = queue.pop(0)
|
|
635
|
+
if cls_key in seen or len(seen) > 12:
|
|
636
|
+
continue
|
|
637
|
+
seen.add(cls_key)
|
|
638
|
+
path, qual = cls_key.split("::", 1)
|
|
639
|
+
wanted = f"{qual}.{method}"
|
|
640
|
+
if any(s["qualified_name"] == wanted for s in per_file_symbols.get(path, [])):
|
|
641
|
+
return f"{path}::{wanted}" + ("" if cls_key == found else "#base")
|
|
642
|
+
queue.extend((bases or {}).get(cls_key, []))
|
|
643
|
+
return None
|
|
644
|
+
|
|
645
|
+
# ── BL-9: RAISES/LOGS/complexity, all single-pass reads of the same tree
|
|
646
|
+
# already parsed for Symbol extraction -- zero new dependency. ─────────────
|
|
647
|
+
_LOG_LEVELS = {"error", "warning", "critical", "exception"}
|
|
648
|
+
_LOG_LEVEL_MAP = {"exception": "error"} # .exception() logs at error level
|
|
649
|
+
_LOGGER_RECEIVERS = {"logger", "logging", "log"}
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def _message_template(node: ast.expr) -> str | None:
|
|
653
|
+
"""A literal or f-string message, interpolated parts normalized to `{}`
|
|
654
|
+
-- same "a message can be long/formatted" handling Warning's own hashing
|
|
655
|
+
already established. None for anything else (a str.format() call, a
|
|
656
|
+
computed value) -- not guessed."""
|
|
657
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
658
|
+
return node.value
|
|
659
|
+
if isinstance(node, ast.JoinedStr):
|
|
660
|
+
parts = [str(v.value) if isinstance(v, ast.Constant) else "{}" for v in node.values]
|
|
661
|
+
return "".join(parts)
|
|
662
|
+
return None
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _exception_type_name(node: ast.expr) -> str | None:
|
|
666
|
+
if isinstance(node, ast.Call):
|
|
667
|
+
func = node.func
|
|
668
|
+
if isinstance(func, ast.Name):
|
|
669
|
+
return func.id
|
|
670
|
+
if isinstance(func, ast.Attribute):
|
|
671
|
+
return func.attr
|
|
672
|
+
if isinstance(node, ast.Name):
|
|
673
|
+
return node.id
|
|
674
|
+
return None
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _http_status_code(call: ast.Call) -> int | None:
|
|
678
|
+
for kw in call.keywords:
|
|
679
|
+
if kw.arg == "status_code" and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, int):
|
|
680
|
+
return kw.value.value
|
|
681
|
+
return None # a variable/expression status_code is skipped, not guessed
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _raises_in_body(node: ast.AST) -> list[tuple[str, str | None, int | None]]:
|
|
685
|
+
"""(exception_type, message_template, status_code) for each `raise
|
|
686
|
+
Exc(...)` directly in node's own body -- same top-scoping as
|
|
687
|
+
_calls_in_body, so a nested function's own raises attribute to ITS
|
|
688
|
+
symbol, not this one."""
|
|
689
|
+
out: list[tuple[str, str | None, int | None]] = []
|
|
690
|
+
|
|
691
|
+
def walk(n: ast.AST, top: bool) -> None:
|
|
692
|
+
for child in ast.iter_child_nodes(n):
|
|
693
|
+
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and not top:
|
|
694
|
+
continue
|
|
695
|
+
if isinstance(child, ast.Raise) and child.exc is not None:
|
|
696
|
+
exc_type = _exception_type_name(child.exc)
|
|
697
|
+
if exc_type:
|
|
698
|
+
message = None
|
|
699
|
+
status_code = None
|
|
700
|
+
if isinstance(child.exc, ast.Call):
|
|
701
|
+
if child.exc.args:
|
|
702
|
+
message = _message_template(child.exc.args[0])
|
|
703
|
+
for kw in child.exc.keywords:
|
|
704
|
+
if kw.arg == "detail": # HTTPException's own message kwarg
|
|
705
|
+
message = _message_template(kw.value) or message
|
|
706
|
+
status_code = _http_status_code(child.exc)
|
|
707
|
+
out.append((exc_type, message, status_code))
|
|
708
|
+
walk(child, top=False)
|
|
709
|
+
|
|
710
|
+
walk(node, top=True)
|
|
711
|
+
return out
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _logs_in_body(node: ast.AST) -> list[tuple[str, str | None]]:
|
|
715
|
+
"""(level, message_template) for recognized logger.<level>(...) calls
|
|
716
|
+
directly in node's own body -- allow-list receiver names, extensible
|
|
717
|
+
the same way _HTTP_METHODS/_ROUTER_NAMES are."""
|
|
718
|
+
out: list[tuple[str, str | None]] = []
|
|
719
|
+
|
|
720
|
+
def walk(n: ast.AST, top: bool) -> None:
|
|
721
|
+
for child in ast.iter_child_nodes(n):
|
|
722
|
+
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and not top:
|
|
723
|
+
continue
|
|
724
|
+
if (isinstance(child, ast.Call) and isinstance(child.func, ast.Attribute)
|
|
725
|
+
and child.func.attr in _LOG_LEVELS
|
|
726
|
+
and isinstance(child.func.value, ast.Name) and child.func.value.id in _LOGGER_RECEIVERS):
|
|
727
|
+
level = _LOG_LEVEL_MAP.get(child.func.attr, child.func.attr)
|
|
728
|
+
message = _message_template(child.args[0]) if child.args else None
|
|
729
|
+
out.append((level, message))
|
|
730
|
+
walk(child, top=False)
|
|
731
|
+
|
|
732
|
+
walk(node, top=True)
|
|
733
|
+
return out
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _complexity(node: ast.AST) -> int:
|
|
737
|
+
"""Cheap branch-count proxy (KNOWLEDGE_SPEC.md Section 3), not a full
|
|
738
|
+
cyclomatic-complexity library -- counts If/For/While/Try anywhere
|
|
739
|
+
within node's own subtree, nested defs included (this is a structural
|
|
740
|
+
size signal for the whole symbol, not a body-scoped call/raise list)."""
|
|
741
|
+
return sum(1 for child in ast.walk(node) if isinstance(child, (ast.If, ast.For, ast.While, ast.Try)))
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def _error_signature_key(kind: str, label: str, message_template: str | None) -> str:
|
|
745
|
+
import hashlib
|
|
746
|
+
basis = f"{kind}:{label}:{message_template or ''}"
|
|
747
|
+
return hashlib.sha256(basis.encode()).hexdigest()[:16]
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
# ── BL-10: middleware registration -- a distinct pattern from a direct
|
|
751
|
+
# CALLS edge (registration, not invocation), same rationale as BL-4b's
|
|
752
|
+
# route capture. ────────────────────────────────────────────────────────
|
|
753
|
+
def _middleware_registrations(tree: ast.Module) -> list[tuple[str, str]]:
|
|
754
|
+
"""(middleware_short_name, receiver) for each `<app_or_router>.
|
|
755
|
+
add_middleware(X, ...)` call anywhere in the file."""
|
|
756
|
+
out: list[tuple[str, str]] = []
|
|
757
|
+
for node in ast.walk(tree):
|
|
758
|
+
if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
|
|
759
|
+
and node.func.attr == "add_middleware"):
|
|
760
|
+
continue
|
|
761
|
+
if not (isinstance(node.func.value, ast.Name) and node.func.value.id in _ROUTER_NAMES):
|
|
762
|
+
continue
|
|
763
|
+
if not node.args:
|
|
764
|
+
continue
|
|
765
|
+
arg = node.args[0]
|
|
766
|
+
name = arg.id if isinstance(arg, ast.Name) else (arg.attr if isinstance(arg, ast.Attribute) else None)
|
|
767
|
+
if name:
|
|
768
|
+
out.append((name, node.func.value.id))
|
|
769
|
+
return out
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
# ── BL-12: daemon-entry touch-point detection (KNOWLEDGE_SPEC.md Section
|
|
773
|
+
# 4a) -- the one touch-point kind nothing else already captures (`scheduled`
|
|
774
|
+
# reads Symbol.decorators, already extracted; `ui_handler`/`endpoint` are
|
|
775
|
+
# derivable from graph structure alone). Extensible, not exhaustive, same
|
|
776
|
+
# framing as the spec's own "new kinds get added as new patterns are
|
|
777
|
+
# found" note. ──────────────────────────────────────────────────────────
|
|
778
|
+
def _daemon_entries(tree: ast.Module) -> list[str]:
|
|
779
|
+
"""Short names of same-file symbols invoked as a process/thread entry
|
|
780
|
+
point: `if __name__ == "__main__": <call>()`, or `target=` to
|
|
781
|
+
`Thread(...)`/passed to `asyncio.create_task(...)`. Same-file
|
|
782
|
+
resolution only -- these patterns overwhelmingly target a local
|
|
783
|
+
function in real code, and a cross-file guess here risks a false
|
|
784
|
+
touch point more than a missed one is worth."""
|
|
785
|
+
out: list[str] = []
|
|
786
|
+
|
|
787
|
+
def _is_main_guard(test: ast.expr) -> bool:
|
|
788
|
+
return (isinstance(test, ast.Compare) and isinstance(test.left, ast.Name)
|
|
789
|
+
and test.left.id == "__name__" and len(test.comparators) == 1
|
|
790
|
+
and isinstance(test.comparators[0], ast.Constant) and test.comparators[0].value == "__main__")
|
|
791
|
+
|
|
792
|
+
for node in ast.walk(tree):
|
|
793
|
+
if isinstance(node, ast.If) and _is_main_guard(node.test):
|
|
794
|
+
for call in ast.walk(node):
|
|
795
|
+
if isinstance(call, ast.Call):
|
|
796
|
+
name = _exception_type_name(call) # reuses the same Call->short-name unwrap
|
|
797
|
+
if name:
|
|
798
|
+
out.append(name)
|
|
799
|
+
elif (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
|
|
800
|
+
and node.func.id in ("Thread", "create_task")):
|
|
801
|
+
for kw in node.keywords:
|
|
802
|
+
if kw.arg == "target" and isinstance(kw.value, ast.Name):
|
|
803
|
+
out.append(kw.value.id)
|
|
804
|
+
for arg in node.args:
|
|
805
|
+
if isinstance(arg, ast.Call) and isinstance(arg.func, ast.Name):
|
|
806
|
+
out.append(arg.func.id)
|
|
807
|
+
return out
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
# ── BL-14: policy/criticality tagging -- fully automated, no hand-
|
|
811
|
+
# maintained field (KNOWLEDGE_SPEC.md Section 3/8 OQ2). Computed once
|
|
812
|
+
# RAISES/REGISTERED_ON are known for a symbol, in the same pass -- no need
|
|
813
|
+
# for a separate post-merge step since both inputs already exist here. ────
|
|
814
|
+
_SECURITY_PATH_KEYWORDS = (
|
|
815
|
+
"auth", "iam", "middleware", "security", "permission", "credential", "secret", "token", "crypto",
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _path_keyword_signals(rel_path: str) -> list[str]:
|
|
820
|
+
lower = rel_path.lower()
|
|
821
|
+
return [f"path-keyword:{kw}" for kw in _SECURITY_PATH_KEYWORDS if kw in lower]
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
class _Visitor(ast.NodeVisitor):
|
|
825
|
+
def __init__(self) -> None:
|
|
826
|
+
self.stack: list[tuple[str, bool]] = [] # (name, is_class)
|
|
827
|
+
self.symbols: list[dict] = []
|
|
828
|
+
self.calls: list[tuple[str, str | None, str]] = [] # (caller_qualname, receiver, callee_short)
|
|
829
|
+
self.routes: list[tuple[str, str, str]] = [] # (qualname, method, path)
|
|
830
|
+
self.raises: list[tuple[str, str, str | None, int | None]] = [] # (qualname, exc_type, message, status_code)
|
|
831
|
+
self.logs: list[tuple[str, str, str | None]] = [] # (qualname, level, message)
|
|
832
|
+
self.router_prefix = ""
|
|
833
|
+
|
|
834
|
+
def _qualname(self, name: str) -> str:
|
|
835
|
+
return ".".join([*(n for n, _ in self.stack), name])
|
|
836
|
+
|
|
837
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> None: # noqa: N802
|
|
838
|
+
qn = self._qualname(node.name)
|
|
839
|
+
self.symbols.append({
|
|
840
|
+
"qualified_name": qn, "kind": "class",
|
|
841
|
+
"docstring_summary": _first_line(ast.get_docstring(node)),
|
|
842
|
+
"decorators": [ast.unparse(d) for d in node.decorator_list],
|
|
843
|
+
"is_public": not node.name.startswith("_"),
|
|
844
|
+
"line_start": node.lineno, "line_end": getattr(node, "end_lineno", node.lineno),
|
|
845
|
+
"complexity": None, # not a meaningful branch-count unit for a class itself
|
|
846
|
+
"bases": [b for b in (_class_ref(x) for x in node.bases) if b], # ODSF-313, ADR-SDP-DEVKB-006
|
|
847
|
+
})
|
|
848
|
+
self.stack.append((node.name, True))
|
|
849
|
+
self.generic_visit(node)
|
|
850
|
+
self.stack.pop()
|
|
851
|
+
|
|
852
|
+
def visit_FunctionDef(self, node: ast.FunctionDef) -> None: # noqa: N802
|
|
853
|
+
self._visit_func(node)
|
|
854
|
+
|
|
855
|
+
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: # noqa: N802
|
|
856
|
+
self._visit_func(node)
|
|
857
|
+
|
|
858
|
+
def _visit_func(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
|
|
859
|
+
qn = self._qualname(node.name)
|
|
860
|
+
kind = "method" if self.stack and self.stack[-1][1] else "function"
|
|
861
|
+
self.symbols.append({
|
|
862
|
+
"qualified_name": qn, "kind": kind,
|
|
863
|
+
"docstring_summary": _first_line(ast.get_docstring(node)),
|
|
864
|
+
"decorators": [ast.unparse(d) for d in node.decorator_list],
|
|
865
|
+
"is_public": not node.name.startswith("_"),
|
|
866
|
+
"line_start": node.lineno, "line_end": getattr(node, "end_lineno", node.lineno),
|
|
867
|
+
"complexity": _complexity(node),
|
|
868
|
+
})
|
|
869
|
+
self.calls.extend((qn, receiver, callee) for receiver, callee in _calls_in_body(node))
|
|
870
|
+
self.raises.extend((qn, exc_type, message, status_code)
|
|
871
|
+
for exc_type, message, status_code in _raises_in_body(node))
|
|
872
|
+
self.logs.extend((qn, level, message) for level, message in _logs_in_body(node))
|
|
873
|
+
for dec in node.decorator_list:
|
|
874
|
+
route = _decorator_route(dec)
|
|
875
|
+
if route:
|
|
876
|
+
method, path = route
|
|
877
|
+
full_path = f"{self.router_prefix}{path}" if self.router_prefix and not path.startswith(self.router_prefix) else path
|
|
878
|
+
self.routes.append((qn, method, full_path))
|
|
879
|
+
self.stack.append((node.name, False))
|
|
880
|
+
self.generic_visit(node)
|
|
881
|
+
self.stack.pop()
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def _parse_python_file(
|
|
885
|
+
path: Path,
|
|
886
|
+
) -> tuple[list[dict], list[tuple[str, str | None, str]], list[tuple[str, str | None]],
|
|
887
|
+
list[tuple[str, str, str]], str | None, list[tuple[str, str, str | None, int | None]],
|
|
888
|
+
list[tuple[str, str, str | None]], list[tuple[str, str]], str, list[str],
|
|
889
|
+
list[tuple[int, str, str | None, str]], list[str], list[tuple[str, str, str]]]:
|
|
890
|
+
"""Returns (symbols, calls, imports, routes, module_docstring, raises,
|
|
891
|
+
logs, middleware_registrations, router_prefix, daemon_entries,
|
|
892
|
+
import_descriptors, path_literals, typed_calls) for one file. An unparseable file yields all-empty,
|
|
893
|
+
not a crash."""
|
|
894
|
+
try:
|
|
895
|
+
text = path.read_text(errors="replace")
|
|
896
|
+
tree = ast.parse(text, filename=str(path))
|
|
897
|
+
except (SyntaxError, OSError, ValueError, RecursionError):
|
|
898
|
+
return [], [], [], [], None, [], [], [], "", [], [], [], []
|
|
899
|
+
v = _Visitor()
|
|
900
|
+
v.router_prefix = _router_prefix(tree)
|
|
901
|
+
v.visit(tree)
|
|
902
|
+
v.calls.extend((MODULE_CALLER, receiver, callee) for receiver, callee in _module_level_calls(tree))
|
|
903
|
+
module_docstring = ast.get_docstring(tree)
|
|
904
|
+
return (v.symbols, v.calls, _imports(tree), v.routes, module_docstring, v.raises, v.logs,
|
|
905
|
+
_middleware_registrations(tree), v.router_prefix, _daemon_entries(tree), _import_descriptors(tree), _py_literals(tree),
|
|
906
|
+
_typed_calls(tree))
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
def extract(repo_root: Path) -> DevKBGraph:
|
|
910
|
+
g = DevKBGraph()
|
|
911
|
+
|
|
912
|
+
# ── Pass 1 (Python): parse every file, collect symbols/calls/imports/
|
|
913
|
+
# routes before resolving anything -- cross-file resolution needs every
|
|
914
|
+
# file's own symbol table to already exist. ────────────────────────────
|
|
915
|
+
per_file_symbols: dict[str, list[dict]] = {}
|
|
916
|
+
per_file_calls: dict[str, list[tuple[str, str | None, str]]] = {}
|
|
917
|
+
per_file_imports: dict[str, dict[str, str | None]] = {}
|
|
918
|
+
per_file_literals: dict[str, list[str]] = {}
|
|
919
|
+
per_file_import_targets: dict[str, tuple[dict[str, list[str]], dict[str, tuple[str, str]]]] = {}
|
|
920
|
+
per_file_docstring: dict[str, str | None] = {}
|
|
921
|
+
per_file_raises: dict[str, list[tuple[str, str, str | None, int | None]]] = {}
|
|
922
|
+
per_file_logs: dict[str, list[tuple[str, str, str | None]]] = {}
|
|
923
|
+
per_file_middleware: dict[str, list[tuple[str, str]]] = {}
|
|
924
|
+
per_file_router_prefix: dict[str, str] = {}
|
|
925
|
+
per_file_daemon_entries: dict[str, list[str]] = {}
|
|
926
|
+
per_file_typed_calls: dict[str, list[tuple[str, str, str]]] = {}
|
|
927
|
+
route_index: dict[str, str] = {} # "METHOD /path" -> Symbol natural_key
|
|
928
|
+
reexports: dict[str, dict[str, tuple[str, str]]] = {} # every parsed file's from-imports, symbols or not
|
|
929
|
+
module_callers: dict[str, set[str]] = {} # called Symbol -> files that call it outside any function
|
|
930
|
+
|
|
931
|
+
for rel_path in _list_python_files(repo_root):
|
|
932
|
+
(symbols, calls, imports, routes, module_doc, raises, logs, middleware, router_prefix,
|
|
933
|
+
daemon_entries, descriptors, literals, typed_calls) = _parse_python_file(repo_root / rel_path)
|
|
934
|
+
# `@typing.overload` signatures and the implementation share one qualified name, and so one Symbol key: keep
|
|
935
|
+
# one entry (the last, the implementation), or every lookup by name sees several and resolves nothing
|
|
936
|
+
# (click's `prompt`: all 13 call sites were left unattributed).
|
|
937
|
+
symbols = list({s["qualified_name"]: s for s in symbols}.values())
|
|
938
|
+
# Keep the file even with zero functions/classes when it still has
|
|
939
|
+
# something real to attach -- a bare entrypoint (`app.py`: imports
|
|
940
|
+
# + `app.add_middleware(...)` calls, no def/class at all) would
|
|
941
|
+
# otherwise silently lose its own middleware registrations here,
|
|
942
|
+
# a real bug this exact shape caught live (BL-10's own test).
|
|
943
|
+
targets = _import_targets(descriptors, rel_path)
|
|
944
|
+
if targets[1]:
|
|
945
|
+
reexports[rel_path] = targets[1]
|
|
946
|
+
if not symbols and not middleware and not module_doc and not daemon_entries and not calls:
|
|
947
|
+
continue
|
|
948
|
+
per_file_symbols[rel_path] = symbols
|
|
949
|
+
per_file_calls[rel_path] = calls
|
|
950
|
+
per_file_imports[rel_path] = dict(imports)
|
|
951
|
+
per_file_import_targets[rel_path] = targets
|
|
952
|
+
per_file_literals[rel_path] = literals
|
|
953
|
+
per_file_docstring[rel_path] = module_doc
|
|
954
|
+
per_file_raises[rel_path] = raises
|
|
955
|
+
per_file_logs[rel_path] = logs
|
|
956
|
+
per_file_middleware[rel_path] = middleware
|
|
957
|
+
per_file_router_prefix[rel_path] = router_prefix
|
|
958
|
+
per_file_daemon_entries[rel_path] = daemon_entries
|
|
959
|
+
per_file_typed_calls[rel_path] = typed_calls
|
|
960
|
+
for qn, method, path in routes:
|
|
961
|
+
key, owner = f"{method} {path}", f"{rel_path}::{qn}"
|
|
962
|
+
if route_index.get(key, owner) != owner:
|
|
963
|
+
# Two handlers under one "METHOD path" (routers mounted under prefixes this file cannot see): keep
|
|
964
|
+
# both. The first keeps the plain key, which CALLS_API matches; the second is told apart by its file.
|
|
965
|
+
key = f"{method} {path} ({rel_path})"
|
|
966
|
+
route_index[key] = owner
|
|
967
|
+
|
|
968
|
+
register_reexports(per_file_symbols, reexports)
|
|
969
|
+
|
|
970
|
+
# ── BL-7: persist Endpoint as a real, independently-queryable node --
|
|
971
|
+
# previously route_index only lived as an in-memory resolution
|
|
972
|
+
# intermediate for CALLS_API below, thrown away once built. Natural key
|
|
973
|
+
# matches discovery/'s own "METHOD path" convention exactly, so the two
|
|
974
|
+
# stores stay joinable without a translation table. ─────────────────
|
|
975
|
+
for route_key, owner_symbol in route_index.items():
|
|
976
|
+
method, path = route_key.split(" ", 1)
|
|
977
|
+
path = path.split(" (", 1)[0] # a disambiguated key carries its file; the path does not
|
|
978
|
+
g.add_node("Endpoint", route_key, source="ast", method=method, path=path, owner_symbol=owner_symbol)
|
|
979
|
+
|
|
980
|
+
# security_signals (BL-14) accumulates across RAISES/REGISTERED_ON
|
|
981
|
+
# below, keyed by the owning Symbol's natural_key -- applied to the
|
|
982
|
+
# Symbol node once fully known, at the end of this pass.
|
|
983
|
+
security_signals: dict[str, list[str]] = {}
|
|
984
|
+
|
|
985
|
+
for rel_path, symbols in per_file_symbols.items():
|
|
986
|
+
g.add_node("File", rel_path, source="ast", description=per_file_docstring.get(rel_path))
|
|
987
|
+
daemon_entries = set(per_file_daemon_entries.get(rel_path, []))
|
|
988
|
+
for s in symbols:
|
|
989
|
+
natural_key = f"{rel_path}::{s['qualified_name']}"
|
|
990
|
+
is_daemon_entry = s["qualified_name"].rsplit(".", 1)[-1] in daemon_entries
|
|
991
|
+
g.add_node(
|
|
992
|
+
"Symbol", natural_key, source="ast",
|
|
993
|
+
kind=s["kind"], docstring_summary=s["docstring_summary"],
|
|
994
|
+
decorators=s["decorators"], is_public=s["is_public"],
|
|
995
|
+
line_start=s["line_start"], line_end=s["line_end"],
|
|
996
|
+
complexity=s.get("complexity"),
|
|
997
|
+
is_daemon_entry=is_daemon_entry or None, # add_node strips None -- absent means "no"
|
|
998
|
+
)
|
|
999
|
+
g.add_edge(
|
|
1000
|
+
"DEFINED_IN", "Symbol", natural_key, "File", rel_path,
|
|
1001
|
+
source="ast", confidence=1.0, evidence_ref=f"{rel_path}:{s['line_start']}",
|
|
1002
|
+
)
|
|
1003
|
+
for kw in _path_keyword_signals(rel_path):
|
|
1004
|
+
security_signals.setdefault(natural_key, []).append(kw)
|
|
1005
|
+
|
|
1006
|
+
# ── BL-9: RAISES + LOGS -> ErrorSignature. Same node type for both
|
|
1007
|
+
# sources (a log line and a raised exception with matching text
|
|
1008
|
+
# resolve to the same real-world error), status_code only ever set for
|
|
1009
|
+
# the HTTPException-literal-int case (RAISES). ─────────────────────
|
|
1010
|
+
for rel_path, raises in per_file_raises.items():
|
|
1011
|
+
for qn, exc_type, message, status_code in raises:
|
|
1012
|
+
caller_key = f"{rel_path}::{qn}"
|
|
1013
|
+
sig_key = _error_signature_key("raised", exc_type, message)
|
|
1014
|
+
g.add_node("ErrorSignature", sig_key, source="ast", kind="raised", label=exc_type,
|
|
1015
|
+
message_template=message, status_code=status_code)
|
|
1016
|
+
confidence = 1.0 if status_code is not None else 0.9
|
|
1017
|
+
g.add_edge(
|
|
1018
|
+
"RAISES", "Symbol", caller_key, "ErrorSignature", sig_key,
|
|
1019
|
+
source="ast", confidence=confidence,
|
|
1020
|
+
evidence_ref=f"{rel_path}::{qn} raises {exc_type}",
|
|
1021
|
+
)
|
|
1022
|
+
if status_code in (401, 403):
|
|
1023
|
+
security_signals.setdefault(caller_key, []).append("enforces-authz-boundary")
|
|
1024
|
+
|
|
1025
|
+
for rel_path, logs in per_file_logs.items():
|
|
1026
|
+
for qn, level, message in logs:
|
|
1027
|
+
caller_key = f"{rel_path}::{qn}"
|
|
1028
|
+
sig_key = _error_signature_key("logged", level, message)
|
|
1029
|
+
g.add_node("ErrorSignature", sig_key, source="ast", kind="logged", label=level,
|
|
1030
|
+
message_template=message, status_code=None)
|
|
1031
|
+
g.add_edge(
|
|
1032
|
+
"LOGS", "Symbol", caller_key, "ErrorSignature", sig_key,
|
|
1033
|
+
source="ast", confidence=0.85,
|
|
1034
|
+
evidence_ref=f"{rel_path}::{qn} logs.{level}(...)",
|
|
1035
|
+
)
|
|
1036
|
+
|
|
1037
|
+
# ── BL-10: REGISTERED_ON -- a registration pattern, structurally
|
|
1038
|
+
# different from a direct call. Resolves the middleware argument to a
|
|
1039
|
+
# real Symbol (same-file, then cross-file via this file's own import,
|
|
1040
|
+
# same precedent as CALLS) and targets a wildcard Endpoint (this file's
|
|
1041
|
+
# own router_prefix, or the bare app) since middleware wraps every
|
|
1042
|
+
# route under it, not one. ──────────────────────────────────────────
|
|
1043
|
+
for rel_path, middleware in per_file_middleware.items():
|
|
1044
|
+
if not middleware:
|
|
1045
|
+
continue
|
|
1046
|
+
symbols = per_file_symbols[rel_path]
|
|
1047
|
+
local_by_short: dict[str, list[str]] = {}
|
|
1048
|
+
bare_by_short: dict[str, list[str]] = {} # what a bare `name()` can mean: never a method, only `self.name()` is
|
|
1049
|
+
method_by_short: dict[str, list[str]] = {} # what `self.name()` / `cls.name()` can mean in this file
|
|
1050
|
+
for s in symbols:
|
|
1051
|
+
short = s["qualified_name"].rsplit(".", 1)[-1]
|
|
1052
|
+
local_by_short.setdefault(short, []).append(s["qualified_name"])
|
|
1053
|
+
(method_by_short if s.get("kind") == "method" else bare_by_short).setdefault(short, []).append(s["qualified_name"])
|
|
1054
|
+
imports = per_file_imports.get(rel_path, {})
|
|
1055
|
+
|
|
1056
|
+
for mw_name, receiver in middleware:
|
|
1057
|
+
resolved_key = None
|
|
1058
|
+
local_targets = local_by_short.get(mw_name, [])
|
|
1059
|
+
if len(local_targets) == 1:
|
|
1060
|
+
resolved_key = f"{rel_path}::{local_targets[0]}"
|
|
1061
|
+
else:
|
|
1062
|
+
dotted_module = imports.get(mw_name)
|
|
1063
|
+
if dotted_module:
|
|
1064
|
+
resolved_key = _resolve_cross_file_symbol(dotted_module, rel_path, mw_name, per_file_symbols)
|
|
1065
|
+
if resolved_key is None:
|
|
1066
|
+
continue # unresolved middleware target -- skip, don't guess
|
|
1067
|
+
|
|
1068
|
+
# Middleware wraps every route under this router/app, not one --
|
|
1069
|
+
# a wildcard marker keyed by this file's own router_prefix, with
|
|
1070
|
+
# rel_path as a disambiguating fallback for a bare `app` (no
|
|
1071
|
+
# prefix) so two different files' own app-level registrations
|
|
1072
|
+
# don't collide onto the same wildcard node.
|
|
1073
|
+
wildcard_path = per_file_router_prefix.get(rel_path) or rel_path
|
|
1074
|
+
endpoint_key = f"* {wildcard_path}"
|
|
1075
|
+
g.add_node("Endpoint", endpoint_key, source="ast-middleware-wildcard",
|
|
1076
|
+
method="*", path=wildcard_path, owner_symbol=None)
|
|
1077
|
+
g.add_edge(
|
|
1078
|
+
"REGISTERED_ON", "Symbol", resolved_key, "Endpoint", endpoint_key,
|
|
1079
|
+
source="ast", confidence=0.9,
|
|
1080
|
+
evidence_ref=f"{rel_path}: {receiver}.add_middleware({mw_name})",
|
|
1081
|
+
)
|
|
1082
|
+
security_signals.setdefault(resolved_key, []).append("registered-as-middleware")
|
|
1083
|
+
|
|
1084
|
+
for natural_key, signals in security_signals.items():
|
|
1085
|
+
g.add_node("Symbol", natural_key, source="ast", security_signals=sorted(set(signals)))
|
|
1086
|
+
|
|
1087
|
+
# ── ODSF-313: class bases, used to resolve a method call through inheritance into an ordinary CALLS edge. The
|
|
1088
|
+
# CONTAINS/EXTENDS edges ADR-SDP-DEVKB-006 proposes are NOT emitted while it is Proposed: no new verb without an
|
|
1089
|
+
# Accepted ADR (ADR-SDP-DEVKB-001). Calls resolved through a base carry source ast-typed-receiver-base, so the
|
|
1090
|
+
# audit (ODSF-314) can count exactly what inheritance adds. ─────────────────────────────────────────────
|
|
1091
|
+
class_bases: dict[str, list[str]] = {}
|
|
1092
|
+
for rel_path, symbols in per_file_symbols.items():
|
|
1093
|
+
targets = per_file_import_targets.get(rel_path, ({}, {}))
|
|
1094
|
+
for s in symbols:
|
|
1095
|
+
qn = s["qualified_name"]
|
|
1096
|
+
if s["kind"] == "class":
|
|
1097
|
+
for base in s.get("bases") or []:
|
|
1098
|
+
base_key = _resolve_class(base, rel_path, per_file_symbols, targets)
|
|
1099
|
+
if base_key and base_key != f"{rel_path}::{qn}":
|
|
1100
|
+
class_bases.setdefault(f"{rel_path}::{qn}", []).append(base_key)
|
|
1101
|
+
|
|
1102
|
+
# Every call site the resolver leaves unattributed, by the called name, when some symbol in the repository has
|
|
1103
|
+
# that name: the coverage a list answer must report instead of looking complete (ODSF-313).
|
|
1104
|
+
repo_short_names = {s["qualified_name"].rsplit(".", 1)[-1] for symbols in per_file_symbols.values() for s in symbols}
|
|
1105
|
+
unresolved: set[tuple[str, str, str]] = set() # (file, caller qualname, called name)
|
|
1106
|
+
|
|
1107
|
+
# ── Pass 2 (Python, BL-3b): resolve calls -- same-file first (0.8,
|
|
1108
|
+
# unchanged from BL-3), then cross-file via THIS file's own import of
|
|
1109
|
+
# the receiver/callee (0.7 -- one hop of real, evidence-backed
|
|
1110
|
+
# indirection, still not a guess). No project-wide "unique short name
|
|
1111
|
+
# anywhere" fallback -- too easy for a common name (get, run, main) to
|
|
1112
|
+
# silently resolve to the wrong file; missing a match beats
|
|
1113
|
+
# mis-attributing one. ──────────────────────────────────────────────
|
|
1114
|
+
for rel_path, calls in per_file_calls.items():
|
|
1115
|
+
symbols = per_file_symbols[rel_path]
|
|
1116
|
+
local_by_short: dict[str, list[str]] = {}
|
|
1117
|
+
bare_by_short: dict[str, list[str]] = {} # what a bare `name()` can mean: never a method, only `self.name()` is
|
|
1118
|
+
method_by_short: dict[str, list[str]] = {} # what `self.name()` / `cls.name()` can mean in this file
|
|
1119
|
+
for s in symbols:
|
|
1120
|
+
short = s["qualified_name"].rsplit(".", 1)[-1]
|
|
1121
|
+
local_by_short.setdefault(short, []).append(s["qualified_name"])
|
|
1122
|
+
(method_by_short if s.get("kind") == "method" else bare_by_short).setdefault(short, []).append(s["qualified_name"])
|
|
1123
|
+
imports = per_file_imports.get(rel_path, {})
|
|
1124
|
+
|
|
1125
|
+
receivers_here = per_file_import_targets.get(rel_path, ({}, {}))[0]
|
|
1126
|
+
receivers_dotted = per_file_import_targets.get(rel_path, ({}, {}))[0]
|
|
1127
|
+
for caller_qn, receiver, callee_short in calls:
|
|
1128
|
+
caller_key = f"{rel_path}::{caller_qn}"
|
|
1129
|
+
dotted_hit = None
|
|
1130
|
+
if receiver and "." in receiver and receiver != EXPR:
|
|
1131
|
+
root, rest = receiver.split(".", 1)
|
|
1132
|
+
for module in receivers_dotted.get(root, []):
|
|
1133
|
+
dotted_hit = _resolve_cross_file_symbol(f"{module}.{rest}", rel_path, callee_short, per_file_symbols)
|
|
1134
|
+
if dotted_hit:
|
|
1135
|
+
break
|
|
1136
|
+
if dotted_hit is None:
|
|
1137
|
+
receiver = EXPR # a chain not through an imported module: an expression, as before
|
|
1138
|
+
if dotted_hit:
|
|
1139
|
+
if caller_qn == MODULE_CALLER:
|
|
1140
|
+
module_callers.setdefault(dotted_hit, set()).add(rel_path)
|
|
1141
|
+
else:
|
|
1142
|
+
g.add_edge("CALLS", "Symbol", caller_key, "Symbol", dotted_hit, source="ast-cross-file-import",
|
|
1143
|
+
confidence=0.7, evidence_ref=f"{rel_path}::{caller_qn} calls {receiver}.{callee_short}")
|
|
1144
|
+
continue
|
|
1145
|
+
|
|
1146
|
+
# `mod.f()` on an imported name calls the import's `f`, never a same-named function of this file: a
|
|
1147
|
+
# handler `create_x` calling `models.create_x(...)` was recorded as calling itself, and the real chain
|
|
1148
|
+
# into the models layer was lost.
|
|
1149
|
+
# Same file: a bare `f()` is a function or class of this file; `self.f()` / `cls.f()` a method of it. A call on
|
|
1150
|
+
# any other object (`path.resolve()`, an imported module) is never a same-file function: the typed-receiver
|
|
1151
|
+
# and import passes decide those, and `as_path.resolve()` inside `def resolve` was recorded as recursion.
|
|
1152
|
+
local_targets = (method_by_short.get(callee_short, []) if receiver in ("self", "cls") else []) if receiver \
|
|
1153
|
+
else bare_by_short.get(callee_short, [])
|
|
1154
|
+
if len(local_targets) == 1 and caller_qn == MODULE_CALLER:
|
|
1155
|
+
module_callers.setdefault(f"{rel_path}::{local_targets[0]}", set()).add(rel_path)
|
|
1156
|
+
continue
|
|
1157
|
+
if len(local_targets) == 1:
|
|
1158
|
+
g.add_edge(
|
|
1159
|
+
"CALLS", "Symbol", caller_key, "Symbol", f"{rel_path}::{local_targets[0]}",
|
|
1160
|
+
source="ast-same-file", confidence=0.8,
|
|
1161
|
+
evidence_ref=f"{rel_path}::{caller_qn} calls {callee_short}",
|
|
1162
|
+
)
|
|
1163
|
+
continue
|
|
1164
|
+
|
|
1165
|
+
receivers, bare = per_file_import_targets.get(rel_path, ({}, {}))
|
|
1166
|
+
resolved_key = None
|
|
1167
|
+
# ODSF-305: resolve through what the file actually imported --
|
|
1168
|
+
# a module alias, `from pkg import module`, a relative import.
|
|
1169
|
+
if receiver:
|
|
1170
|
+
for module in receivers.get(receiver, []):
|
|
1171
|
+
resolved_key = _resolve_cross_file_symbol(module, rel_path, callee_short, per_file_symbols)
|
|
1172
|
+
if resolved_key:
|
|
1173
|
+
break
|
|
1174
|
+
elif callee_short in bare:
|
|
1175
|
+
module, original = bare[callee_short]
|
|
1176
|
+
resolved_key = _resolve_cross_file_symbol(module, rel_path, original, per_file_symbols)
|
|
1177
|
+
if resolved_key is None and receiver and receiver not in receivers and receiver.isidentifier():
|
|
1178
|
+
# A module loaded with importlib and bound to a variable of the same name.
|
|
1179
|
+
resolved_key = _resolve_cross_file_symbol(receiver, rel_path, callee_short, per_file_symbols)
|
|
1180
|
+
if receiver == EXPR:
|
|
1181
|
+
if callee_short in repo_short_names:
|
|
1182
|
+
unresolved.add((rel_path, caller_qn, callee_short)) # said in coverage, as before: not attributed
|
|
1183
|
+
continue
|
|
1184
|
+
if resolved_key is None and receiver and receiver not in receivers:
|
|
1185
|
+
resolved_key = _resolve_via_literals(per_file_literals.get(rel_path, []), callee_short, per_file_symbols)
|
|
1186
|
+
if resolved_key is None:
|
|
1187
|
+
dotted_module = imports.get(receiver) if receiver else imports.get(callee_short)
|
|
1188
|
+
if dotted_module is None and receiver is None and callee_short not in imports:
|
|
1189
|
+
if callee_short in repo_short_names and len(local_targets) != 1:
|
|
1190
|
+
unresolved.add((rel_path, caller_qn, callee_short))
|
|
1191
|
+
continue # nothing to resolve against
|
|
1192
|
+
if dotted_module:
|
|
1193
|
+
resolved_key = _resolve_cross_file_symbol(dotted_module, rel_path, callee_short, per_file_symbols)
|
|
1194
|
+
if resolved_key and caller_qn == MODULE_CALLER:
|
|
1195
|
+
module_callers.setdefault(resolved_key, set()).add(rel_path)
|
|
1196
|
+
elif resolved_key:
|
|
1197
|
+
g.add_edge(
|
|
1198
|
+
"CALLS", "Symbol", caller_key, "Symbol", resolved_key,
|
|
1199
|
+
source="ast-cross-file-import", confidence=0.7,
|
|
1200
|
+
evidence_ref=f"{rel_path}::{caller_qn} calls {receiver + '.' if receiver else ''}{callee_short}",
|
|
1201
|
+
)
|
|
1202
|
+
elif callee_short in repo_short_names:
|
|
1203
|
+
unresolved.add((rel_path, caller_qn, callee_short))
|
|
1204
|
+
|
|
1205
|
+
# ── ODSF-312: calls on an instance of a class the code names ────────────
|
|
1206
|
+
for rel_path, typed in per_file_typed_calls.items():
|
|
1207
|
+
targets = per_file_import_targets.get(rel_path, ({}, {}))
|
|
1208
|
+
for caller_qn, cls_ref, method in typed:
|
|
1209
|
+
resolved_key = _resolve_typed_call(cls_ref, method, rel_path, per_file_symbols, targets, class_bases)
|
|
1210
|
+
via_base = bool(resolved_key and resolved_key.endswith("#base"))
|
|
1211
|
+
resolved_key = resolved_key[: -len("#base")] if via_base else resolved_key
|
|
1212
|
+
if resolved_key and resolved_key != f"{rel_path}::{caller_qn}":
|
|
1213
|
+
g.add_edge(
|
|
1214
|
+
"CALLS", "Symbol", f"{rel_path}::{caller_qn}", "Symbol", resolved_key,
|
|
1215
|
+
source="ast-typed-receiver-base" if via_base else "ast-typed-receiver", confidence=0.6,
|
|
1216
|
+
evidence_ref=f"{rel_path}::{caller_qn} calls {method} on a {cls_ref.replace('self:', '')}",
|
|
1217
|
+
)
|
|
1218
|
+
unresolved.discard((rel_path, caller_qn, method))
|
|
1219
|
+
|
|
1220
|
+
by_name: dict[str, dict[str, int]] = {}
|
|
1221
|
+
for rel_path, _caller, name in unresolved:
|
|
1222
|
+
by_name.setdefault(name, {})[rel_path] = by_name.setdefault(name, {}).get(rel_path, 0) + 1
|
|
1223
|
+
for rel_path, symbols in per_file_symbols.items():
|
|
1224
|
+
for s in symbols:
|
|
1225
|
+
files = by_name.get(s["qualified_name"].rsplit(".", 1)[-1])
|
|
1226
|
+
if files:
|
|
1227
|
+
ranked = sorted(files, key=lambda f: (-files[f], f))
|
|
1228
|
+
g.add_node("Symbol", f"{rel_path}::{s['qualified_name']}", source="ast",
|
|
1229
|
+
unresolved_calls=sum(files.values()), unresolved_files=ranked[:20],
|
|
1230
|
+
unresolved_file_count=len(ranked))
|
|
1231
|
+
for key, files in module_callers.items():
|
|
1232
|
+
g.add_node("Symbol", key, source="ast", module_callers=sorted(files))
|
|
1233
|
+
|
|
1234
|
+
# ── BL-3c: TypeScript/TSX Symbols + their own route-call sites ─────────
|
|
1235
|
+
ts_route_calls: list[tuple[str, str, str, str]] = [] # (file, symbol_qn, method, path)
|
|
1236
|
+
ts_files = _list_ts_files(repo_root)
|
|
1237
|
+
helpers = ts_symbols.collect_url_helpers([repo_root / rel for rel in ts_files]) # ODSF-314: builders, wrappers
|
|
1238
|
+
for rel_path in ts_files:
|
|
1239
|
+
symbols, route_calls = ts_symbols.extract_file(repo_root / rel_path, repo_root, helpers)
|
|
1240
|
+
if not symbols:
|
|
1241
|
+
continue
|
|
1242
|
+
g.add_node("File", rel_path, source="ast-ts")
|
|
1243
|
+
for s in symbols:
|
|
1244
|
+
natural_key = f"{rel_path}::{s['qualified_name']}"
|
|
1245
|
+
g.add_node(
|
|
1246
|
+
"Symbol", natural_key, source="ast-ts",
|
|
1247
|
+
kind=s["kind"], docstring_summary=None, decorators=[], is_public=True,
|
|
1248
|
+
line_start=s["line_start"], line_end=s["line_start"],
|
|
1249
|
+
)
|
|
1250
|
+
g.add_edge(
|
|
1251
|
+
"DEFINED_IN", "Symbol", natural_key, "File", rel_path,
|
|
1252
|
+
source="ast-ts", confidence=1.0, evidence_ref=f"{rel_path}:{s['line_start']}",
|
|
1253
|
+
)
|
|
1254
|
+
for symbol_qn, method, path in route_calls:
|
|
1255
|
+
ts_route_calls.append((rel_path, symbol_qn, method, path))
|
|
1256
|
+
|
|
1257
|
+
# ── ODSF-304: Odoo XML declarations (templates, views, actions, menus) ──
|
|
1258
|
+
for rel_path, decls in _odoo_xml_symbols(repo_root).items():
|
|
1259
|
+
g.add_node("File", rel_path, source="odoo-xml")
|
|
1260
|
+
for d in decls:
|
|
1261
|
+
natural_key = f"{rel_path}::{d['qualified_name']}"
|
|
1262
|
+
g.add_node(
|
|
1263
|
+
"Symbol", natural_key, source="odoo-xml",
|
|
1264
|
+
kind=d["kind"], docstring_summary=None, decorators=[], is_public=True,
|
|
1265
|
+
line_start=d["line_start"], line_end=d["line_start"],
|
|
1266
|
+
)
|
|
1267
|
+
g.add_edge(
|
|
1268
|
+
"DEFINED_IN", "Symbol", natural_key, "File", rel_path,
|
|
1269
|
+
source="odoo-xml", confidence=1.0, evidence_ref=f"{rel_path}:{d['line_start']}",
|
|
1270
|
+
)
|
|
1271
|
+
|
|
1272
|
+
# ── BL-4b: resolve each frontend route-call against the real backend
|
|
1273
|
+
# Symbol that handles it -- CALLS_API, the actual UI-action-to-code
|
|
1274
|
+
# connective tissue. Unresolved (no matching backend route found, e.g.
|
|
1275
|
+
# the path has a runtime prefix this scan can't see) is left
|
|
1276
|
+
# unresolved, not guessed. ──────────────────────────────────────────
|
|
1277
|
+
# ODSF-314: compare route shapes, so `/jobs/{job_id}` matches a call built as `/jobs/${id}`; a call whose method
|
|
1278
|
+
# the extractor could not read matches a path with exactly one route.
|
|
1279
|
+
by_shape: dict[tuple[str, str], str] = {}
|
|
1280
|
+
by_path: dict[str, list[str]] = {}
|
|
1281
|
+
for route_key, owner in route_index.items():
|
|
1282
|
+
m, p = route_key.split(" ", 1)
|
|
1283
|
+
shape = ts_symbols.route_shape(p.split(" (", 1)[0])
|
|
1284
|
+
# a path two handlers serve is ambiguous: a call to it stays unresolved rather than picking one
|
|
1285
|
+
by_shape[(m, shape)] = owner if by_shape.get((m, shape), owner) == owner else None
|
|
1286
|
+
by_path.setdefault(shape, []).append(owner)
|
|
1287
|
+
for file_path, symbol_qn, method, path in ts_route_calls:
|
|
1288
|
+
target = route_index.get(f"{method} {path}") or by_shape.get((method, ts_symbols.route_shape(path)))
|
|
1289
|
+
if target is None and len(set(by_path.get(ts_symbols.route_shape(path), []))) == 1:
|
|
1290
|
+
target = by_path[ts_symbols.route_shape(path)][0]
|
|
1291
|
+
if target is None:
|
|
1292
|
+
continue
|
|
1293
|
+
g.add_edge(
|
|
1294
|
+
"CALLS_API", "Symbol", f"{file_path}::{symbol_qn}", "Symbol", target,
|
|
1295
|
+
source="ast-route-match", confidence=0.85,
|
|
1296
|
+
evidence_ref=f"{file_path}::{symbol_qn} calls {method} {path}",
|
|
1297
|
+
)
|
|
1298
|
+
|
|
1299
|
+
return g
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
def load_cache(ctx: ProjectContext) -> DevKBGraph:
|
|
1303
|
+
"""Reconstructs a DevKBGraph from `.devkb/symbols-{nodes,edges}.jsonl`
|
|
1304
|
+
-- the regular devkb/build.py calls this to merge BL-3's already-built
|
|
1305
|
+
Symbol graph into the main store, never regenerating it itself. Missing
|
|
1306
|
+
cache (no refresh run yet) returns an empty graph, not an error -- same
|
|
1307
|
+
"can't check yet" framing used elsewhere in devkb.
|
|
1308
|
+
|
|
1309
|
+
Node dicts keep their own natural_key, so those round-trip directly.
|
|
1310
|
+
Edge dicts only persist from_uid/to_uid (the hashed form, per
|
|
1311
|
+
Edge.to_dict()), so reconstructing a real add_edge() call needs a
|
|
1312
|
+
uid -> (type, natural_key) reverse lookup built from the nodes pass
|
|
1313
|
+
first -- an edge referencing a uid missing from that map (a corrupt or
|
|
1314
|
+
hand-edited cache) is skipped, not a crash."""
|
|
1315
|
+
g = DevKBGraph()
|
|
1316
|
+
nodes_path = ctx.devkb_dir / "symbols-nodes.jsonl"
|
|
1317
|
+
edges_path = ctx.devkb_dir / "symbols-edges.jsonl"
|
|
1318
|
+
if not nodes_path.exists():
|
|
1319
|
+
return g
|
|
1320
|
+
|
|
1321
|
+
uid_to_key: dict[str, tuple[str, str]] = {}
|
|
1322
|
+
for line in nodes_path.read_text().splitlines():
|
|
1323
|
+
line = line.strip()
|
|
1324
|
+
if not line:
|
|
1325
|
+
continue
|
|
1326
|
+
n = json.loads(line)
|
|
1327
|
+
g.add_node(n["type"], n["natural_key"], source=n.get("source", "cache"), **n.get("properties", {}))
|
|
1328
|
+
uid_to_key[n["uid"]] = (n["type"], n["natural_key"])
|
|
1329
|
+
|
|
1330
|
+
if edges_path.exists():
|
|
1331
|
+
for line in edges_path.read_text().splitlines():
|
|
1332
|
+
line = line.strip()
|
|
1333
|
+
if not line:
|
|
1334
|
+
continue
|
|
1335
|
+
e = json.loads(line)
|
|
1336
|
+
src = uid_to_key.get(e["from_uid"])
|
|
1337
|
+
dst = uid_to_key.get(e["to_uid"])
|
|
1338
|
+
if src is None or dst is None:
|
|
1339
|
+
continue
|
|
1340
|
+
props = e.get("properties", {})
|
|
1341
|
+
extra = {k: v for k, v in props.items() if k not in ("source", "confidence", "evidence_ref")}
|
|
1342
|
+
g.add_edge(
|
|
1343
|
+
e["verb"], src[0], src[1], dst[0], dst[1],
|
|
1344
|
+
source=props.get("source", "cache"), confidence=props.get("confidence", 1.0),
|
|
1345
|
+
evidence_ref=props.get("evidence_ref", ""), extra=extra,
|
|
1346
|
+
)
|
|
1347
|
+
return g
|
|
1348
|
+
|
|
1349
|
+
|
|
1350
|
+
def build(code: str) -> tuple[DevKBGraph, Path]:
|
|
1351
|
+
ctx = resolve(code)
|
|
1352
|
+
graph = extract(ctx.repo_root)
|
|
1353
|
+
|
|
1354
|
+
ctx.devkb_dir.mkdir(parents=True, exist_ok=True)
|
|
1355
|
+
data = graph.to_dict()
|
|
1356
|
+
|
|
1357
|
+
nodes_path = ctx.devkb_dir / "symbols-nodes.jsonl"
|
|
1358
|
+
edges_path = ctx.devkb_dir / "symbols-edges.jsonl"
|
|
1359
|
+
manifest_path = ctx.devkb_dir / "symbols-manifest.json"
|
|
1360
|
+
|
|
1361
|
+
nodes_path.write_text("\n".join(json.dumps(n) for n in data["nodes"]) + ("\n" if data["nodes"] else ""))
|
|
1362
|
+
edges_path.write_text("\n".join(json.dumps(e) for e in data["edges"]) + ("\n" if data["edges"] else ""))
|
|
1363
|
+
manifest_path.write_text(json.dumps({
|
|
1364
|
+
"project": code,
|
|
1365
|
+
"built_at": datetime.now(timezone.utc).isoformat(),
|
|
1366
|
+
"git_head": git_head(ctx.repo_root),
|
|
1367
|
+
"node_count": graph.node_count(),
|
|
1368
|
+
"edge_count": graph.edge_count(),
|
|
1369
|
+
}, indent=2))
|
|
1370
|
+
|
|
1371
|
+
return graph, ctx.devkb_dir
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
def main() -> int:
|
|
1375
|
+
ap = argparse.ArgumentParser(description="Refresh devkb's Symbol graph for one project (BL-3, standalone)")
|
|
1376
|
+
ap.add_argument("--project", required=True, help="Project code, e.g. ODSF (see projects.json)")
|
|
1377
|
+
# `devkb build` hands the same arguments to this and to build.py; the GitHub layer belongs to the latter.
|
|
1378
|
+
ap.add_argument("--github", action="store_true", default=None, help=argparse.SUPPRESS)
|
|
1379
|
+
ap.add_argument("--no-github", dest="github", action="store_false", help=argparse.SUPPRESS)
|
|
1380
|
+
args = ap.parse_args()
|
|
1381
|
+
|
|
1382
|
+
try:
|
|
1383
|
+
graph, devkb_dir = build(args.project)
|
|
1384
|
+
except ValueError as exc:
|
|
1385
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
1386
|
+
return 1
|
|
1387
|
+
|
|
1388
|
+
print(f"devkb symbol refresh ({args.project}): {graph.node_count()} nodes, {graph.edge_count()} edges "
|
|
1389
|
+
f"-> {devkb_dir}")
|
|
1390
|
+
return 0
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
if __name__ == "__main__":
|
|
1394
|
+
sys.exit(main())
|