mapwright-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapwright/__init__.py +3 -0
- mapwright/__main__.py +3 -0
- mapwright/analyze.py +809 -0
- mapwright/brief.py +208 -0
- mapwright/cli.py +596 -0
- mapwright/concepts.py +190 -0
- mapwright/config.py +204 -0
- mapwright/connect.py +337 -0
- mapwright/export.py +42 -0
- mapwright/firestore.py +168 -0
- mapwright/flow.py +238 -0
- mapwright/hooks.py +111 -0
- mapwright/ignore.py +248 -0
- mapwright/infra.py +253 -0
- mapwright/mcp.py +153 -0
- mapwright/noise.py +80 -0
- mapwright/redact.py +39 -0
- mapwright/render.py +428 -0
- mapwright/routes_js.py +282 -0
- mapwright/server.py +388 -0
- mapwright/titles.py +111 -0
- mapwright/track.py +715 -0
- mapwright/ui/app.css +277 -0
- mapwright/ui/app.js +1025 -0
- mapwright/ui/icons.svg +73 -0
- mapwright/ui/index.html +35 -0
- mapwright/ui/mark.svg +1 -0
- mapwright/ui/vendor/mermaid.LICENSE +21 -0
- mapwright/ui/vendor/mermaid.min.js +2029 -0
- mapwright_agent-0.2.0.dist-info/METADATA +179 -0
- mapwright_agent-0.2.0.dist-info/RECORD +34 -0
- mapwright_agent-0.2.0.dist-info/WHEEL +4 -0
- mapwright_agent-0.2.0.dist-info/entry_points.txt +3 -0
- mapwright_agent-0.2.0.dist-info/licenses/LICENSE +21 -0
mapwright/analyze.py
ADDED
|
@@ -0,0 +1,809 @@
|
|
|
1
|
+
"""Static analysis: turn the files of a project into one structured model.
|
|
2
|
+
|
|
3
|
+
Python goes through `ast`; JS/TS, SQL and Prisma through patterns. Nothing in the project is
|
|
4
|
+
imported or executed. Per-file results are cached by content hash, so re-analysing after an
|
|
5
|
+
agent edit costs one file, not the whole tree.
|
|
6
|
+
|
|
7
|
+
The model holds modules, classes, functions (with calls, complexity and algorithm hints),
|
|
8
|
+
HTTP routes, tables, layers, concepts (see concepts.py) and infrastructure (see infra.py).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
from collections import Counter
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from mapwright import firestore as _fs
|
|
23
|
+
from mapwright import routes_js as _rjs
|
|
24
|
+
|
|
25
|
+
IGNORE_DIRS = {".git", "node_modules", ".venv", "venv", "env", "__pycache__", "dist", "build", ".tox", ".mypy_cache",
|
|
26
|
+
".pytest_cache", ".ruff_cache", ".next", ".nuxt", "target", "vendor", ".idea", ".vscode", "coverage",
|
|
27
|
+
".cache", "site-packages", ".eggs", ".terraform", ".gradle", "htmlcov", ".mapwright", ".svelte-kit", ".turbo",
|
|
28
|
+
".parcel-cache", ".output", "out", "bower_components", ".serverless"}
|
|
29
|
+
CODE_EXT = {".py": "python", ".js": "js", ".jsx": "js", ".mjs": "js", ".cjs": "js", ".ts": "js", ".tsx": "js",
|
|
30
|
+
".sql": "sql", ".prisma": "prisma", ".rules": "rules"}
|
|
31
|
+
TEXT_EXT = set(CODE_EXT) | {".md", ".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".html", ".css", ".scss", ".sh",
|
|
32
|
+
".go", ".rs", ".java", ".rb", ".php", ".c", ".h", ".cpp", ".txt", ".graphql", ".proto",
|
|
33
|
+
".kt", ".swift", ".cs", ".vue", ".svelte", ".tf", ".xml"}
|
|
34
|
+
TEXT_NAMES = {"Makefile", "Dockerfile", "Procfile", "Jenkinsfile", ".gitignore", ".dockerignore", "Containerfile",
|
|
35
|
+
"Gemfile", "requirements.txt", "go.mod", "Cargo.toml", ".env.example", ".env.sample", ".env.template"}
|
|
36
|
+
MAX_FILES, MAX_FILE_BYTES, MAX_CACHE_CHARS, MAX_DIFF_LINES = 5000, 256_000, 32_000_000, 120
|
|
37
|
+
def _secretish(rel: str) -> bool:
|
|
38
|
+
name = os.path.basename(rel).lower()
|
|
39
|
+
if name.startswith(".env"):
|
|
40
|
+
return not name.endswith((".example", ".sample", ".template", ".dist"))
|
|
41
|
+
return (name.endswith((".pem", ".key", ".p12", ".pfx", ".keystore")) or "id_rsa" in name or "id_ed25519" in name
|
|
42
|
+
or name in (".netrc", ".npmrc", ".pypirc", "credentials", "secrets.yaml", "secrets.yml", "secrets.json"))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _texty(rel: str) -> bool:
|
|
46
|
+
return os.path.splitext(rel)[1].lower() in TEXT_EXT or os.path.basename(rel) in TEXT_NAMES
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _sha(text: str) -> str:
|
|
50
|
+
return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def find_root(cwd: str) -> str:
|
|
54
|
+
"""The project root: the nearest ancestor holding .git, else the directory itself."""
|
|
55
|
+
p = Path(cwd)
|
|
56
|
+
for d in [p, *p.parents][:6]:
|
|
57
|
+
if (d / ".git").exists():
|
|
58
|
+
return str(d)
|
|
59
|
+
return str(p)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def scan(root: str, matcher=None, max_files: int | None = None, info: dict | None = None) -> tuple[dict[str, tuple[int, int]], bool]:
|
|
63
|
+
"""rel path -> (mtime_ns, size) for every file worth tracking, in sorted order. Second value: hit the file cap.
|
|
64
|
+
|
|
65
|
+
`matcher` (ignore.Matcher) adds user/.gitignore exclusions; no module state is touched. When `info` is given it
|
|
66
|
+
is filled with {"seen", "cap", "top"}: candidate files counted past the cap and top-level dirs by files kept.
|
|
67
|
+
"""
|
|
68
|
+
cap = max_files if max_files else MAX_FILES
|
|
69
|
+
out: dict[str, tuple[int, int]] = {}
|
|
70
|
+
seen, truncated = 0, False
|
|
71
|
+
for dirpath, dirs, files in os.walk(root):
|
|
72
|
+
rd = os.path.relpath(dirpath, root).replace(os.sep, "/")
|
|
73
|
+
rd = "" if rd == "." else rd
|
|
74
|
+
dirs[:] = sorted(d for d in dirs if d not in IGNORE_DIRS and not d.endswith(".egg-info")
|
|
75
|
+
and not (matcher and matcher.matches(f"{rd}/{d}" if rd else d, True)))
|
|
76
|
+
for f in sorted(files):
|
|
77
|
+
rel = f"{rd}/{f}" if rd else f
|
|
78
|
+
if not (_texty(rel) or _secretish(rel)):
|
|
79
|
+
continue
|
|
80
|
+
if matcher and matcher.matches(rel, False):
|
|
81
|
+
continue
|
|
82
|
+
seen += 1
|
|
83
|
+
if truncated:
|
|
84
|
+
continue
|
|
85
|
+
try:
|
|
86
|
+
st = os.stat(os.path.join(dirpath, f))
|
|
87
|
+
except OSError:
|
|
88
|
+
seen -= 1
|
|
89
|
+
continue
|
|
90
|
+
out[rel] = (st.st_mtime_ns, st.st_size)
|
|
91
|
+
if len(out) >= cap:
|
|
92
|
+
truncated = True
|
|
93
|
+
if info is not None:
|
|
94
|
+
top = Counter(r.split("/")[0] if "/" in r else "(root)" for r in out)
|
|
95
|
+
info.clear()
|
|
96
|
+
info.update({"seen": seen, "cap": cap, "top": [{"dir": d, "files": n} for d, n in top.most_common(5)]})
|
|
97
|
+
return out, truncated
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def read_text(root: str, rel: str) -> str | None:
|
|
101
|
+
"""File text, or None if it is missing, too large, binary or secret-looking."""
|
|
102
|
+
if _secretish(rel):
|
|
103
|
+
return None
|
|
104
|
+
try:
|
|
105
|
+
with open(os.path.join(root, rel), "rb") as fh:
|
|
106
|
+
raw = fh.read(MAX_FILE_BYTES + 1)
|
|
107
|
+
except OSError:
|
|
108
|
+
return None
|
|
109
|
+
if len(raw) > MAX_FILE_BYTES or b"\0" in raw:
|
|
110
|
+
return None
|
|
111
|
+
return raw.decode("utf-8", "replace")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ============================================================================ per-file analysis
|
|
115
|
+
def _name(node) -> str:
|
|
116
|
+
try:
|
|
117
|
+
return ast.unparse(node)
|
|
118
|
+
except Exception:
|
|
119
|
+
return ""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _py_module_id(rel: str) -> str:
|
|
123
|
+
parts = rel[:-3].split("/")
|
|
124
|
+
if parts[-1] == "__init__":
|
|
125
|
+
parts = parts[:-1] or ["__init__"]
|
|
126
|
+
return ".".join(parts)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _call_name(node) -> str:
|
|
130
|
+
return _name(node.func).split(".")[-1] if isinstance(node, ast.Call) else ""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _py_class(node: ast.ClassDef) -> tuple[dict, dict | None]:
|
|
134
|
+
attrs, methods, table, cols = [], [], None, []
|
|
135
|
+
bases = [_name(b) for b in node.bases if _name(b)]
|
|
136
|
+
django = any(b.split(".")[-1] == "Model" and "models" in b or b == "Model" for b in bases)
|
|
137
|
+
|
|
138
|
+
def column(name: str, value, ann: str = "") -> None:
|
|
139
|
+
fn = _call_name(value)
|
|
140
|
+
if fn in ("Column", "mapped_column"):
|
|
141
|
+
typ = next((_name(a).split("(")[0].split(".")[-1] for a in value.args if _call_name(a) != "ForeignKey"), "") or ann
|
|
142
|
+
fk = next((a.args[0].value for a in value.args if _call_name(a) == "ForeignKey" and a.args
|
|
143
|
+
and isinstance(a.args[0], ast.Constant) and isinstance(a.args[0].value, str)), None)
|
|
144
|
+
pk = any(k.arg == "primary_key" and getattr(k.value, "value", False) for k in value.keywords)
|
|
145
|
+
cols.append({"name": name, "type": typ or "col", "pk": pk, "fk": fk.split(".")[0] if fk else None})
|
|
146
|
+
elif django and (fn.endswith("Field") or fn in ("ForeignKey", "OneToOneField", "ManyToManyField")):
|
|
147
|
+
fk = None
|
|
148
|
+
if fn in ("ForeignKey", "OneToOneField", "ManyToManyField") and value.args:
|
|
149
|
+
a = value.args[0]
|
|
150
|
+
fk = (a.value if isinstance(a, ast.Constant) else _name(a)).split(".")[-1].lower()
|
|
151
|
+
cols.append({"name": name, "type": fn.replace("Field", "") or "Field", "pk": False, "fk": fk})
|
|
152
|
+
|
|
153
|
+
for item in node.body:
|
|
154
|
+
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
|
|
155
|
+
ann = _name(item.annotation)
|
|
156
|
+
attrs.append([item.target.id, ann])
|
|
157
|
+
if item.value is not None:
|
|
158
|
+
column(item.target.id, item.value, re.sub(r"^Mapped\[(.*)\]$", r"\1", ann))
|
|
159
|
+
elif isinstance(item, ast.Assign) and len(item.targets) == 1 and isinstance(item.targets[0], ast.Name):
|
|
160
|
+
n = item.targets[0].id
|
|
161
|
+
if n == "__tablename__" and isinstance(item.value, ast.Constant):
|
|
162
|
+
table = str(item.value.value)
|
|
163
|
+
elif not n.startswith("__"):
|
|
164
|
+
attrs.append([n, ""])
|
|
165
|
+
column(n, item.value)
|
|
166
|
+
elif isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
167
|
+
methods.append(item.name)
|
|
168
|
+
if item.name == "__init__":
|
|
169
|
+
for sub in ast.walk(item):
|
|
170
|
+
tgt = sub.target if isinstance(sub, ast.AnnAssign) else sub.targets[0] if isinstance(sub, ast.Assign) and sub.targets else None
|
|
171
|
+
if isinstance(tgt, ast.Attribute) and isinstance(tgt.value, ast.Name) and tgt.value.id == "self":
|
|
172
|
+
ann = _name(sub.annotation) if isinstance(sub, ast.AnnAssign) else ""
|
|
173
|
+
if tgt.attr not in [a[0] for a in attrs]:
|
|
174
|
+
attrs.append([tgt.attr, ann])
|
|
175
|
+
cls = {"name": node.name, "line": node.lineno, "bases": bases, "attrs": attrs[:40], "methods": methods[:60]}
|
|
176
|
+
if django and not table:
|
|
177
|
+
table = node.name.lower()
|
|
178
|
+
if not any(c["pk"] for c in cols):
|
|
179
|
+
cols.insert(0, {"name": "id", "type": "Auto", "pk": True, "fk": None})
|
|
180
|
+
if table and cols:
|
|
181
|
+
return cls, {"name": table, "line": node.lineno, "columns": cols, "source": "django" if django else "sqlalchemy"}
|
|
182
|
+
return cls, None
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def analyze_python(rel: str, text: str) -> dict:
|
|
186
|
+
out = {"lang": "python", "id": _py_module_id(rel), "loc": text.count("\n") + 1, "imports": [], "classes": [], "tables": [],
|
|
187
|
+
"functions": [], "routes": [], "raw_imports": [], "aliases": {}}
|
|
188
|
+
try:
|
|
189
|
+
tree = ast.parse(text)
|
|
190
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
191
|
+
return {**out, "error": True} # the agent may be half-way through an edit
|
|
192
|
+
parts = rel[:-3].split("/")
|
|
193
|
+
pkg = parts if parts[-1] == "__init__" else parts[:-1]
|
|
194
|
+
if parts[-1] == "__init__":
|
|
195
|
+
pkg = parts[:-1]
|
|
196
|
+
for node in ast.walk(tree):
|
|
197
|
+
if isinstance(node, ast.Import):
|
|
198
|
+
out["imports"] += [[a.name] for a in node.names]
|
|
199
|
+
out["raw_imports"] += [a.name for a in node.names]
|
|
200
|
+
for a in node.names:
|
|
201
|
+
if a.asname:
|
|
202
|
+
out["aliases"][a.asname] = [a.name]
|
|
203
|
+
elif isinstance(node, ast.ImportFrom):
|
|
204
|
+
if node.level:
|
|
205
|
+
base = pkg[: len(pkg) - (node.level - 1)] if node.level - 1 <= len(pkg) else []
|
|
206
|
+
mod = ".".join(base + ([node.module] if node.module else []))
|
|
207
|
+
rel_import = True
|
|
208
|
+
else:
|
|
209
|
+
mod, rel_import = node.module or "", False
|
|
210
|
+
out["raw_imports"].append(mod)
|
|
211
|
+
for a in node.names:
|
|
212
|
+
cands = [f"{mod}.{a.name}" if mod else a.name, mod] if a.name != "*" else [mod]
|
|
213
|
+
out["imports"].append([c for c in cands if c] + (["."] if rel_import else []))
|
|
214
|
+
if a.name != "*":
|
|
215
|
+
out["aliases"][a.asname or a.name] = [c for c in cands if c][:1]
|
|
216
|
+
prefixes = {}
|
|
217
|
+
for node in tree.body: # router = APIRouter(prefix="/orders") · bp = Blueprint("x", __name__, url_prefix="/x")
|
|
218
|
+
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call):
|
|
219
|
+
for kw in node.value.keywords:
|
|
220
|
+
if kw.arg in ("prefix", "url_prefix") and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
|
|
221
|
+
prefixes[node.targets[0].id] = kw.value.value.rstrip("/")
|
|
222
|
+
for node in tree.body:
|
|
223
|
+
if isinstance(node, ast.ClassDef):
|
|
224
|
+
cls, table = _py_class(node)
|
|
225
|
+
cls["end"] = getattr(node, "end_lineno", node.lineno)
|
|
226
|
+
out["classes"].append(cls)
|
|
227
|
+
if table:
|
|
228
|
+
cls["table"] = table["name"]
|
|
229
|
+
out["tables"].append(table)
|
|
230
|
+
for item in node.body:
|
|
231
|
+
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
232
|
+
out["functions"].append(_py_func(item, text, cls_name=node.name))
|
|
233
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
234
|
+
out["functions"].append(_py_func(node, text))
|
|
235
|
+
out["raw_imports"] = sorted({m for m in out["raw_imports"] if m})
|
|
236
|
+
for node in ast.walk(tree): # CREATE TABLE inside string literals (sqlite3 scripts, migrations in code)
|
|
237
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str) and re.search(r"(?i)create\s+table", node.value):
|
|
238
|
+
for t in analyze_sql(rel, node.value)["tables"]:
|
|
239
|
+
if t["name"] not in {x["name"] for x in out["tables"]}:
|
|
240
|
+
out["tables"].append({**t, "line": node.lineno + t["line"] - 1, "source": "sql in code"})
|
|
241
|
+
for f in out["functions"]:
|
|
242
|
+
for r in f.pop("_routes", []):
|
|
243
|
+
path = (prefixes.get(r["via"], "") + "/" + r["path"].lstrip("/")).rstrip("/") or "/" if r["via"] in prefixes else r["path"]
|
|
244
|
+
out["routes"].append({**r, "path": path, "handler": f["qual"], "line": f["line"]})
|
|
245
|
+
out["sql_tables"] = sorted(set(_SQL_REF.findall(text)))[:40]
|
|
246
|
+
return out
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
_JS_IMPORT = re.compile(r"""(?:\bimport\s+(?:[^'"()]*?\sfrom\s+)?|\bexport\s+[^'"()]*?\sfrom\s+|\brequire\(\s*|\bimport\(\s*)['"]([^'"\n]+)['"]""")
|
|
250
|
+
_JS_CLASS = re.compile(r"^\s*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)(?:\s+extends\s+([\w$.]+))?", re.M)
|
|
251
|
+
_JS_IFACE = re.compile(r"^\s*(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)(?:\s+extends\s+([\w$., ]+?))?\s*\{", re.M)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def analyze_js(rel: str, text: str) -> dict:
|
|
255
|
+
out = {"lang": "js", "id": os.path.splitext(rel)[0], "loc": text.count("\n") + 1, "imports": [], "classes": [], "tables": [],
|
|
256
|
+
"functions": [], "routes": [], "raw_imports": sorted(set(_JS_IMPORT.findall(text)))}
|
|
257
|
+
here = os.path.dirname(rel)
|
|
258
|
+
for spec in dict.fromkeys(_JS_IMPORT.findall(text)):
|
|
259
|
+
if spec.startswith("."):
|
|
260
|
+
target = os.path.normpath(os.path.join(here, spec))
|
|
261
|
+
target = re.sub(r"\.(?:js|jsx|ts|tsx|mjs|cjs)$", "", target)
|
|
262
|
+
out["imports"].append([target, target + "/index", "."])
|
|
263
|
+
elif not spec.startswith(("node:", "/", "http")):
|
|
264
|
+
out["imports"].append(["/".join(spec.split("/")[: 2 if spec.startswith("@") else 1])])
|
|
265
|
+
line_of = lambda m: text.count("\n", 0, m.start()) + 1 # noqa: E731
|
|
266
|
+
for m in _JS_CLASS.finditer(text):
|
|
267
|
+
out["classes"].append({"name": m.group(1), "line": line_of(m), "bases": [m.group(2)] if m.group(2) else [],
|
|
268
|
+
"attrs": [], "methods": []})
|
|
269
|
+
for m in _JS_IFACE.finditer(text):
|
|
270
|
+
bases = [b.strip() for b in (m.group(2) or "").split(",") if b.strip()]
|
|
271
|
+
out["classes"].append({"name": m.group(1), "line": line_of(m), "bases": bases, "attrs": [], "methods": [], "kind": "interface"})
|
|
272
|
+
_js_functions(rel, text, out)
|
|
273
|
+
for m in re.finditer(r"(`|\"|')((?:(?!\1)[\s\S]){0,4000}?(?i:create\s+table)[\s\S]{0,4000}?)\1", text):
|
|
274
|
+
for t in analyze_sql(rel, m.group(2))["tables"]:
|
|
275
|
+
if t["name"] not in {x["name"] for x in out["tables"]}:
|
|
276
|
+
out["tables"].append({**t, "line": text.count("\n", 0, m.start()) + t["line"], "source": "sql in code"})
|
|
277
|
+
out["sql_tables"] = sorted(set(_SQL_REF.findall(text)))[:40]
|
|
278
|
+
_js_extras(rel, text, out)
|
|
279
|
+
return out
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _js_extras(rel: str, text: str, out: dict) -> None:
|
|
283
|
+
"""Client routes (react-router), Firestore collections, zod schemas and React tags: see routes_js / firestore."""
|
|
284
|
+
out["routes"] += _rjs.find_routes(rel, text)
|
|
285
|
+
usage = _fs.find_usage(text)
|
|
286
|
+
have = {t["name"] for t in out["tables"]}
|
|
287
|
+
out["tables"] += [t for t in _fs.usage_tables(text) if t["name"] not in have]
|
|
288
|
+
_fs.function_refs(usage, out["functions"])
|
|
289
|
+
out["tables"] += [t for t in _rjs.zod_schemas(text) if t["name"] not in have]
|
|
290
|
+
_rjs.tag_react(rel, text, out["functions"])
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
_SQL_TABLE = re.compile(r"create\s+table\s+(?:if\s+not\s+exists\s+)?[`\"\[]?([\w.]+)[`\"\]]?\s*\(", re.I)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _split_top(body: str) -> list[str]:
|
|
297
|
+
parts, depth, cur = [], 0, []
|
|
298
|
+
for ch in body:
|
|
299
|
+
if ch == "(":
|
|
300
|
+
depth += 1
|
|
301
|
+
elif ch == ")":
|
|
302
|
+
depth -= 1
|
|
303
|
+
if ch == "," and depth == 0:
|
|
304
|
+
parts.append("".join(cur))
|
|
305
|
+
cur = []
|
|
306
|
+
else:
|
|
307
|
+
cur.append(ch)
|
|
308
|
+
parts.append("".join(cur))
|
|
309
|
+
return [p.strip() for p in parts if p.strip()]
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def analyze_sql(rel: str, text: str) -> dict:
|
|
313
|
+
out = {"lang": "sql", "id": rel, "loc": text.count("\n") + 1, "imports": [], "classes": [], "tables": [],
|
|
314
|
+
"functions": [], "routes": [], "raw_imports": []}
|
|
315
|
+
clean = re.sub(r"--[^\n]*", "", text)
|
|
316
|
+
for m in _SQL_TABLE.finditer(clean):
|
|
317
|
+
depth, i = 1, m.end()
|
|
318
|
+
while i < len(clean) and depth:
|
|
319
|
+
depth += {"(": 1, ")": -1}.get(clean[i], 0)
|
|
320
|
+
i += 1
|
|
321
|
+
cols, pks = [], set()
|
|
322
|
+
for part in _split_top(clean[m.end(): i - 1]):
|
|
323
|
+
low = part.lower()
|
|
324
|
+
fkm = re.match(r"(?:constraint\s+\S+\s+)?foreign\s+key\s*\(\s*[`\"]?(\w+)[`\"]?\s*\)\s*references\s+[`\"]?([\w.]+)", low)
|
|
325
|
+
pkm = re.match(r"(?:constraint\s+\S+\s+)?primary\s+key\s*\(([^)]*)\)", low)
|
|
326
|
+
if fkm:
|
|
327
|
+
for c in cols:
|
|
328
|
+
if c["name"].lower() == fkm.group(1):
|
|
329
|
+
c["fk"] = fkm.group(2).split(".")[-1]
|
|
330
|
+
elif pkm:
|
|
331
|
+
pks |= {x.strip(" `\"") for x in pkm.group(1).split(",")}
|
|
332
|
+
elif not re.match(r"(constraint|unique|check|index|key)\b", low):
|
|
333
|
+
toks = part.split()
|
|
334
|
+
if len(toks) >= 2:
|
|
335
|
+
ref = re.search(r"references\s+[`\"]?([\w.]+)", low)
|
|
336
|
+
cols.append({"name": toks[0].strip("`\"[]"), "type": re.sub(r"\(.*", "", toks[1]).strip(",") or "col",
|
|
337
|
+
"pk": "primary key" in low, "fk": ref.group(1).split(".")[-1] if ref else None})
|
|
338
|
+
for c in cols:
|
|
339
|
+
c["pk"] = c["pk"] or c["name"].lower() in pks
|
|
340
|
+
if cols:
|
|
341
|
+
out["tables"].append({"name": m.group(1).split(".")[-1].strip("`\""), "line": clean.count("\n", 0, m.start()) + 1,
|
|
342
|
+
"columns": cols, "source": "sql"})
|
|
343
|
+
return out
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
_PRISMA = re.compile(r"^model\s+(\w+)\s*\{(.*?)^\}", re.M | re.S)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def analyze_prisma(rel: str, text: str) -> dict:
|
|
350
|
+
out = {"lang": "prisma", "id": rel, "loc": text.count("\n") + 1, "imports": [], "classes": [], "tables": [],
|
|
351
|
+
"functions": [], "routes": [], "raw_imports": []}
|
|
352
|
+
models = {m.group(1) for m in _PRISMA.finditer(text)}
|
|
353
|
+
for m in _PRISMA.finditer(text):
|
|
354
|
+
cols = []
|
|
355
|
+
for line in m.group(2).splitlines():
|
|
356
|
+
toks = line.split("//")[0].split()
|
|
357
|
+
if len(toks) < 2 or toks[0].startswith("@@"):
|
|
358
|
+
continue
|
|
359
|
+
typ = toks[1].rstrip("?[]")
|
|
360
|
+
if typ in models:
|
|
361
|
+
if "@relation" in line and "fields:" in line: # the owning side names the relation
|
|
362
|
+
cols.append({"name": toks[0], "type": typ, "pk": False, "fk": typ})
|
|
363
|
+
continue
|
|
364
|
+
cols.append({"name": toks[0], "type": typ, "pk": "@id" in line, "fk": None})
|
|
365
|
+
out["tables"].append({"name": m.group(1), "line": text.count("\n", 0, m.start()) + 1, "columns": cols, "source": "prisma"})
|
|
366
|
+
return out
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
ANALYZERS = {"python": analyze_python, "js": analyze_js, "sql": analyze_sql, "prisma": analyze_prisma, "rules": _fs.analyze_rules}
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# ============================================================================ functions, routes, hints
|
|
375
|
+
_SQL_REF = re.compile(r"(?i)\b(?:from|join|into|update|table)\s+[`\"\[]?([A-Za-z_][\w]*)[`\"\]]?(?=[\s;(,)]|$)")
|
|
376
|
+
_ROUTE_DECO = re.compile(r"""^([\w.]+)\.(get|post|put|patch|delete|head|options|route|websocket|api_route)\(\s*(?:path\s*=\s*)?['"]([^'"]*)['"](.*)$""", re.S)
|
|
377
|
+
_SQL_KW = {"select", "where", "set", "values", "the", "a", "an", "it", "this", "that", "each", "which"}
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _complexity(node) -> int:
|
|
381
|
+
n = 1
|
|
382
|
+
for sub in ast.walk(node):
|
|
383
|
+
if isinstance(sub, (ast.If, ast.For, ast.AsyncFor, ast.While, ast.IfExp, ast.ExceptHandler, ast.Assert)):
|
|
384
|
+
n += 1
|
|
385
|
+
elif isinstance(sub, ast.BoolOp):
|
|
386
|
+
n += len(sub.values) - 1
|
|
387
|
+
elif isinstance(sub, ast.comprehension):
|
|
388
|
+
n += 1 + len(sub.ifs)
|
|
389
|
+
elif sys.version_info >= (3, 10) and isinstance(sub, ast.match_case):
|
|
390
|
+
n += 1
|
|
391
|
+
return n
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def algorithm_hints(names: set[str], calls: set[str], text: str, own: str, is_async: bool, decos: list[str]) -> list[str]:
|
|
395
|
+
"""Recognisable algorithms and mechanisms in one function, from the names it uses.
|
|
396
|
+
Hints, not proofs: the UI and the brief label them as such."""
|
|
397
|
+
low = text.lower()
|
|
398
|
+
h = []
|
|
399
|
+
if own in calls:
|
|
400
|
+
h.append("recursion")
|
|
401
|
+
if any(d.split("(")[0].split(".")[-1] in ("lru_cache", "cache", "cached", "memoize") for d in decos) or "memo" in names:
|
|
402
|
+
h.append("memoization")
|
|
403
|
+
if {"sorted", "sort"} & calls:
|
|
404
|
+
h.append("sorting")
|
|
405
|
+
if "heapq" in low or {"heappush", "heappop"} & calls:
|
|
406
|
+
h.append("priority queue (heap)")
|
|
407
|
+
if "bisect" in low or ({"lo", "hi", "mid"} <= names) or ({"low", "high", "mid"} <= names) or ({"left", "right", "mid"} <= names):
|
|
408
|
+
h.append("binary search")
|
|
409
|
+
if ("popleft" in calls or "shift" in calls) and ("visited" in names or "seen" in names):
|
|
410
|
+
h.append("breadth-first search")
|
|
411
|
+
elif ("visited" in names or "seen" in names) and ("stack" in names or own in calls):
|
|
412
|
+
h.append("depth-first search")
|
|
413
|
+
if "dp" in names or "table" in names and "range" in calls and "min" in calls | {"max"}:
|
|
414
|
+
h.append("dynamic programming")
|
|
415
|
+
if {"sha256", "sha1", "md5", "blake2b", "hexdigest", "createHash"} & calls or "hashlib" in low:
|
|
416
|
+
h.append("hashing")
|
|
417
|
+
if {"compile", "match", "search", "findall", "sub", "fullmatch"} & calls and ("re." in low or "regex" in low):
|
|
418
|
+
h.append("regular expressions")
|
|
419
|
+
if {"sleep"} & calls and any(w in low for w in ("retry", "attempt", "backoff", "tries")):
|
|
420
|
+
h.append("retry with backoff")
|
|
421
|
+
if {"limit", "offset"} <= names or "cursor" in names and "next" in names:
|
|
422
|
+
h.append("pagination")
|
|
423
|
+
if "chunk" in low or "batch" in low:
|
|
424
|
+
h.append("batching")
|
|
425
|
+
if {"execute", "executemany", "query", "raw"} & calls or re.search(r"\bselect\b.+\bfrom\b", low):
|
|
426
|
+
h.append("database query")
|
|
427
|
+
if {"get", "post", "put", "delete", "request", "fetch", "urlopen"} & calls and any(w in low for w in ("http", "requests", "httpx", "fetch(", "url", "axios")):
|
|
428
|
+
h.append("HTTP call")
|
|
429
|
+
if is_async:
|
|
430
|
+
h.append("async I/O")
|
|
431
|
+
if {"gather", "create_task", "ThreadPoolExecutor", "submit", "Thread", "Pool", "all"} & calls and any(w in low for w in ("gather", "thread", "pool", "promise.all", "executor")):
|
|
432
|
+
h.append("concurrency")
|
|
433
|
+
if {"lock", "Lock", "acquire", "Semaphore"} & calls or "with lock" in low:
|
|
434
|
+
h.append("locking")
|
|
435
|
+
if "yield" in low and re.search(r"\byield\b", text):
|
|
436
|
+
h.append("generator / streaming")
|
|
437
|
+
return list(dict.fromkeys(h))
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _py_func(node, text: str, cls_name: str | None = None) -> dict:
|
|
441
|
+
decos = [_name(d) for d in node.decorator_list]
|
|
442
|
+
calls, names = set(), set()
|
|
443
|
+
body_nodes = [x for st in node.body for x in ast.walk(st)] + list(ast.walk(node.args))
|
|
444
|
+
for sub in body_nodes:
|
|
445
|
+
if isinstance(sub, ast.Call):
|
|
446
|
+
nm = _name(sub.func)
|
|
447
|
+
if nm and len(nm) < 80 and "(" not in nm and "[" not in nm:
|
|
448
|
+
segs = nm.split(".")
|
|
449
|
+
calls.add(nm if len(segs) == 1 else ".".join(segs[-2:])) # bare `f`, `self.f`, `obj.f`
|
|
450
|
+
elif isinstance(sub, ast.Name):
|
|
451
|
+
names.add(sub.id)
|
|
452
|
+
elif isinstance(sub, ast.arg):
|
|
453
|
+
names.add(sub.arg)
|
|
454
|
+
elif isinstance(sub, ast.Attribute):
|
|
455
|
+
names.add(sub.attr)
|
|
456
|
+
end = getattr(node, "end_lineno", node.lineno)
|
|
457
|
+
seg = "\n".join(text.splitlines()[min([d.lineno for d in node.decorator_list] + [node.lineno]) - 1:end])
|
|
458
|
+
doc = (ast.get_docstring(node) or "").strip().split("\n")[0][:120]
|
|
459
|
+
args = [a.arg for a in node.args.args if a.arg not in ("self", "cls")]
|
|
460
|
+
if node.args.vararg:
|
|
461
|
+
args.append("*" + node.args.vararg.arg)
|
|
462
|
+
if node.args.kwarg:
|
|
463
|
+
args.append("**" + node.args.kwarg.arg)
|
|
464
|
+
is_async = isinstance(node, ast.AsyncFunctionDef)
|
|
465
|
+
f = {"name": node.name, "qual": f"{cls_name}.{node.name}" if cls_name else node.name, "cls": cls_name,
|
|
466
|
+
"line": node.lineno, "start": min([d.lineno for d in node.decorator_list] + [node.lineno]), "end": end, "args": args[:12], "decorators": decos[:6], "async": is_async,
|
|
467
|
+
"doc": doc, "calls": sorted(calls)[:80], "complexity": _complexity(node), "loc": end - node.lineno + 1,
|
|
468
|
+
"returns": _name(node.returns) if node.returns else "",
|
|
469
|
+
"refs": sorted(n for n in names if n[:1].isupper())[:30],
|
|
470
|
+
"sql_refs": sorted({t for t in _SQL_REF.findall(seg) if t.lower() not in _SQL_KW})[:20],
|
|
471
|
+
"hints": algorithm_hints(names, {c.split(".")[-1] for c in calls}, seg, node.name, is_async, decos)}
|
|
472
|
+
routes = []
|
|
473
|
+
for d in decos:
|
|
474
|
+
m = _ROUTE_DECO.match(d)
|
|
475
|
+
if m:
|
|
476
|
+
meth = m.group(2).upper()
|
|
477
|
+
if meth in ("ROUTE", "API_ROUTE"):
|
|
478
|
+
mm = re.search(r"methods\s*=\s*\[([^\]]*)\]", m.group(4))
|
|
479
|
+
meths = re.findall(r"['\"](\w+)['\"]", mm.group(1)) if mm else ["GET"]
|
|
480
|
+
else:
|
|
481
|
+
meths = ["WS" if meth == "WEBSOCKET" else meth]
|
|
482
|
+
for x in meths:
|
|
483
|
+
routes.append({"method": x.upper(), "path": m.group(3) or "/", "via": m.group(1)})
|
|
484
|
+
if routes:
|
|
485
|
+
f["_routes"] = routes
|
|
486
|
+
return f
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
_JS_KW = {"if", "for", "while", "switch", "catch", "function", "return", "typeof", "new", "await", "import", "require",
|
|
490
|
+
"super", "constructor", "else", "do", "try", "with", "yield", "delete", "void", "in", "of", "case", "throw"}
|
|
491
|
+
_JS_FN = re.compile(
|
|
492
|
+
r"^[ \t]*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s*\*?\s*([A-Za-z_$][\w$]*)\s*\(([^)]*)\)"
|
|
493
|
+
r"|^[ \t]*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*(?::[^=]+)?=\s*(?:async\s+)?(?:function\b[^(]*\(([^)]*)\)|\(([^)]*)\)\s*(?::[^=]+)?=>|([A-Za-z_$][\w$]*)\s*=>)"
|
|
494
|
+
r"|^[ \t]+(?:public\s+|private\s+|protected\s+|static\s+|async\s+|readonly\s+)*([A-Za-z_$][\w$]*)\s*\(([^)]*)\)\s*(?::\s*[^{;]+)?\{", re.M)
|
|
495
|
+
_JS_CALL = re.compile(r"(?<![\w$.])((?:this\.)?[A-Za-z_$][\w$]*)\s*\(")
|
|
496
|
+
_JS_ROUTE = re.compile(r"""\b(app|router|server|api|routes|r)\.(get|post|put|patch|delete|all|head|options)\(\s*(['"`])([^'"`]+)\3\s*,\s*(?:[^,()]+,\s*)*?(?:async\s+)?([A-Za-z_$][\w$.]*)?""")
|
|
497
|
+
_NEXT_ROUTE = re.compile(r"^\s*export\s+(?:async\s+)?function\s+(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\b", re.M)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _js_body(text: str, start: int) -> tuple[int, int]:
|
|
501
|
+
"""(start, end) offsets of the brace-delimited body that starts at or after `start`; strings and
|
|
502
|
+
comments are skipped roughly. An arrow function without braces ends at the end of its line."""
|
|
503
|
+
i = text.find("{", start)
|
|
504
|
+
nl = text.find("\n", start)
|
|
505
|
+
if i < 0 or (nl >= 0 and i > nl and "=>" in text[start:nl] and not text[start:nl].rstrip().endswith("=>")):
|
|
506
|
+
return start, nl if nl >= 0 else len(text)
|
|
507
|
+
depth, j, n = 0, i, len(text)
|
|
508
|
+
quote = None
|
|
509
|
+
while j < n:
|
|
510
|
+
ch = text[j]
|
|
511
|
+
if quote:
|
|
512
|
+
if ch == "\\":
|
|
513
|
+
j += 1
|
|
514
|
+
elif ch == quote:
|
|
515
|
+
quote = None
|
|
516
|
+
elif ch in "'\"`":
|
|
517
|
+
quote = ch
|
|
518
|
+
elif ch == "/" and text[j:j + 2] == "//":
|
|
519
|
+
j = text.find("\n", j)
|
|
520
|
+
if j < 0:
|
|
521
|
+
break
|
|
522
|
+
elif ch == "/" and text[j:j + 2] == "/*":
|
|
523
|
+
j = text.find("*/", j + 2)
|
|
524
|
+
if j < 0:
|
|
525
|
+
break
|
|
526
|
+
j += 1
|
|
527
|
+
elif ch == "{":
|
|
528
|
+
depth += 1
|
|
529
|
+
elif ch == "}":
|
|
530
|
+
depth -= 1
|
|
531
|
+
if depth == 0:
|
|
532
|
+
return i, j + 1
|
|
533
|
+
j += 1
|
|
534
|
+
return i, n
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _js_functions(rel: str, text: str, out: dict) -> None:
|
|
538
|
+
line_of = lambda pos: text.count("\n", 0, pos) + 1 # noqa: E731
|
|
539
|
+
classes = [(text.count("\n", 0, m.start()) + 1, m.group(1)) for m in _JS_CLASS.finditer(text)]
|
|
540
|
+
seen = set()
|
|
541
|
+
for m in _JS_FN.finditer(text):
|
|
542
|
+
name = m.group(1) or m.group(3) or m.group(7)
|
|
543
|
+
if not name or name in _JS_KW:
|
|
544
|
+
continue
|
|
545
|
+
args = m.group(2) if m.group(1) else (m.group(4) or m.group(5) or m.group(6) or "") if m.group(3) else m.group(8)
|
|
546
|
+
line = line_of(m.start())
|
|
547
|
+
method = bool(m.group(7))
|
|
548
|
+
cls = None
|
|
549
|
+
if method:
|
|
550
|
+
cls = next((c for ln, c in reversed(classes) if ln < line), None)
|
|
551
|
+
if not cls:
|
|
552
|
+
continue # an indented call followed by a block, not a method
|
|
553
|
+
a, b = _js_body(text, m.end())
|
|
554
|
+
seg = text[m.start():b]
|
|
555
|
+
calls = {c.replace("this.", "self.") for c in _JS_CALL.findall(seg) if c.split(".")[-1] not in _JS_KW and c != name} \
|
|
556
|
+
| ({name} if re.search(rf"(?<![\w$.]){re.escape(name)}\s*\(", text[a:b]) else set())
|
|
557
|
+
qual = f"{cls}.{name}" if cls else name
|
|
558
|
+
if qual in seen:
|
|
559
|
+
continue
|
|
560
|
+
seen.add(qual)
|
|
561
|
+
names = set(re.findall(r"[A-Za-z_$][\w$]*", seg))
|
|
562
|
+
is_async = bool(re.search(r"\basync\b", text[max(0, m.start()):m.end()]))
|
|
563
|
+
cx = 1 + len(re.findall(r"\b(?:if|for|while|case|catch)\b|&&|\|\||\?\?|\?(?=[^.:])", seg))
|
|
564
|
+
out["functions"].append({
|
|
565
|
+
"name": name, "qual": qual, "cls": cls, "line": line, "end": line_of(b), "async": is_async,
|
|
566
|
+
"args": [x.strip().split(":")[0].split("=")[0].strip() for x in (args or "").split(",") if x.strip()][:12],
|
|
567
|
+
"decorators": [], "doc": "", "calls": sorted(calls)[:80], "complexity": cx, "loc": line_of(b) - line + 1,
|
|
568
|
+
"sql_refs": sorted({t for t in _SQL_REF.findall(seg) if t.lower() not in _SQL_KW})[:20],
|
|
569
|
+
"returns": "", "hints": algorithm_hints(names, {c.split(".")[-1] for c in calls}, seg, name, is_async, [])})
|
|
570
|
+
for m in _JS_ROUTE.finditer(text):
|
|
571
|
+
out["routes"].append({"method": m.group(2).upper().replace("ALL", "ANY"), "path": m.group(4), "via": m.group(1),
|
|
572
|
+
"handler": (m.group(5) or "").split(".")[-1] or None, "line": line_of(m.start())})
|
|
573
|
+
if re.search(r"(?:^|/)app/(?:.+/)?route\.(?:js|ts)$", rel): # Next.js app router
|
|
574
|
+
path = "/" + "/".join(p for p in rel.split("app/", 1)[1].split("/")[:-1] if not p.startswith("("))
|
|
575
|
+
for m in _NEXT_ROUTE.finditer(text):
|
|
576
|
+
out["routes"].append({"method": m.group(1), "path": path or "/", "via": "next", "handler": m.group(1), "line": line_of(m.start())})
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def analyze_file(rel: str, text: str) -> dict | None:
|
|
580
|
+
lang = CODE_EXT.get(os.path.splitext(rel)[1].lower())
|
|
581
|
+
if not lang:
|
|
582
|
+
return None
|
|
583
|
+
rel = rel.replace(os.sep, "/")
|
|
584
|
+
try:
|
|
585
|
+
part = ANALYZERS[lang](rel, text)
|
|
586
|
+
except Exception: # an analyser bug must never stop tracking
|
|
587
|
+
part = {"lang": lang, "id": rel, "loc": text.count("\n") + 1, "imports": [], "classes": [], "tables": [],
|
|
588
|
+
"functions": [], "routes": [], "raw_imports": [], "error": True}
|
|
589
|
+
part["file"], part["sha"] = rel, _sha(text)
|
|
590
|
+
part["sql_tables"] = [t for t in part.get("sql_tables", []) if t.lower() not in _SQL_KW]
|
|
591
|
+
return part
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
# ============================================================================ layers
|
|
595
|
+
LAYERS = ("interface", "service", "data", "integration", "ui", "core", "util", "config", "test")
|
|
596
|
+
LAYER_TITLES = {"interface": "Interface (API, CLI, handlers)", "service": "Services & domain logic", "data": "Data access & models",
|
|
597
|
+
"integration": "Integrations & clients", "ui": "User interface", "core": "Core", "util": "Utilities",
|
|
598
|
+
"config": "Configuration", "test": "Tests"}
|
|
599
|
+
_LAYER_WORDS = [
|
|
600
|
+
("test", r"(^|/)(tests?|spec|__tests__|e2e)(/|$)|(^|/)test_[^/]*$|_test\.\w+$|\.(spec|test)\.\w+$|(^|/)conftest\.py$"),
|
|
601
|
+
("ui", r"(^|/)(components?|pages|views|ui|frontend|client|app/\(.*\)|layouts|screens|widgets)(/|$)|\.(jsx|tsx|vue|svelte)$"),
|
|
602
|
+
("interface", r"(^|/)(api|routes?|routers?|controllers?|handlers?|endpoints?|cli|commands?|graphql|resolvers?|server|web)(/|$|\.)|(^|/)(main|app|server|cli|wsgi|asgi|manage)\.\w+$"),
|
|
603
|
+
("data", r"(^|/)(models?|db|database|repositor(y|ies)|schemas?|entities|migrations?|orm|dao|store|persistence|prisma)(/|$|\.)"),
|
|
604
|
+
("integration", r"(^|/)(clients?|integrations?|adapters?|gateways?|providers?|external|sdk|connectors?)(/|$|\.)"),
|
|
605
|
+
("service", r"(^|/)(services?|domain|core|logic|usecases?|use_cases|workers?|jobs?|tasks?|engine|pipelines?)(/|$|\.)"),
|
|
606
|
+
("config", r"(^|/)(config|settings|conf|env|constants)(/|$|\.)"),
|
|
607
|
+
("util", r"(^|/)(utils?|helpers?|lib|common|shared|tools)(/|$|\.)"),
|
|
608
|
+
]
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def layer_of(rel: str, part: dict | None = None) -> str:
|
|
612
|
+
low = rel.lower()
|
|
613
|
+
for layer, pat in _LAYER_WORDS:
|
|
614
|
+
if re.search(pat, low):
|
|
615
|
+
return layer
|
|
616
|
+
if part:
|
|
617
|
+
if part.get("routes"):
|
|
618
|
+
return "interface"
|
|
619
|
+
if part.get("tables"):
|
|
620
|
+
return "data"
|
|
621
|
+
return "core"
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
# ============================================================================ linking into one model
|
|
625
|
+
_STDLIB = set(getattr(sys, "stdlib_module_names", ())) | {"__future__"}
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def link(parts: dict[str, dict]) -> dict:
|
|
629
|
+
"""Join per-file results into one model."""
|
|
630
|
+
by_id, suffix = {}, {}
|
|
631
|
+
for rel, p in parts.items():
|
|
632
|
+
by_id[p["id"]] = rel
|
|
633
|
+
for mid in sorted(by_id):
|
|
634
|
+
segs = re.split(r"[./]", mid)
|
|
635
|
+
sep = "/" if "/" in mid and parts[by_id[mid]]["lang"] == "js" else "."
|
|
636
|
+
for i in range(len(segs)):
|
|
637
|
+
suffix.setdefault(sep.join(segs[i:]), mid)
|
|
638
|
+
own = {re.split(r"[./]", s_)[0] for s_ in suffix} | {os.path.basename(os.path.dirname(r)) for r in parts}
|
|
639
|
+
modules, classes, tables, functions, routes = {}, {}, {}, {}, []
|
|
640
|
+
for rel in sorted(parts):
|
|
641
|
+
p = parts[rel]
|
|
642
|
+
deps, ext = set(), set()
|
|
643
|
+
for cands in p["imports"]:
|
|
644
|
+
relative = cands[-1] == "."
|
|
645
|
+
hit = next((suffix[c] for c in cands if c != "." and c in suffix), None)
|
|
646
|
+
if hit and hit != p["id"]:
|
|
647
|
+
deps.add(hit)
|
|
648
|
+
elif not hit and not relative:
|
|
649
|
+
top = cands[-1].split(".")[0] if p["lang"] == "python" else cands[-1]
|
|
650
|
+
if top and top not in _STDLIB and top not in own:
|
|
651
|
+
ext.add(top)
|
|
652
|
+
if p["lang"] in ("python", "js"):
|
|
653
|
+
modules[p["id"]] = {"file": rel, "lang": p["lang"], "pkg": os.path.dirname(rel) or ".", "deps": sorted(deps),
|
|
654
|
+
"ext": sorted(ext), "loc": p["loc"], "sha": p["sha"], "error": bool(p.get("error")),
|
|
655
|
+
"layer": layer_of(rel, p), "functions": [], "classes": [c["name"] for c in p["classes"]],
|
|
656
|
+
"sql_tables": p.get("sql_tables", [])}
|
|
657
|
+
for c in p["classes"]:
|
|
658
|
+
classes[f'{p["id"]}:{c["name"]}'] = {**c, "module": p["id"], "file": rel}
|
|
659
|
+
for t in p["tables"]:
|
|
660
|
+
if t["name"] in tables: # the same table in SQL and in an ORM model: keep the first, note the other
|
|
661
|
+
if t.get("kind") == "firestore":
|
|
662
|
+
_fs.merge(tables[t["name"]], {**t, "file": rel}, rel)
|
|
663
|
+
else:
|
|
664
|
+
tables[t["name"]].setdefault("also_in", []).append(rel)
|
|
665
|
+
else:
|
|
666
|
+
tables[t["name"]] = {**t, "file": rel}
|
|
667
|
+
for f in p.get("functions", []):
|
|
668
|
+
fid = f'{p["id"]}:{f["qual"]}'
|
|
669
|
+
functions[fid] = {**f, "module": p["id"], "file": rel, "callees": [], "callers": []}
|
|
670
|
+
if p["id"] in modules:
|
|
671
|
+
modules[p["id"]]["functions"].append(f["qual"])
|
|
672
|
+
for r in p.get("routes", []):
|
|
673
|
+
routes.append({**r, "module": p["id"], "file": rel})
|
|
674
|
+
|
|
675
|
+
# ---- call graph: resolve call names to project functions
|
|
676
|
+
by_name: dict[str, list[str]] = {}
|
|
677
|
+
for fid, f in functions.items():
|
|
678
|
+
by_name.setdefault(f["name"], []).append(fid)
|
|
679
|
+
mod_tail = {}
|
|
680
|
+
for mid in modules:
|
|
681
|
+
mod_tail.setdefault(re.split(r"[./]", mid)[-1], []).append(mid)
|
|
682
|
+
alias_mod: dict[str, dict[str, str]] = {} # module -> alias -> module id it names
|
|
683
|
+
for rel, p in parts.items():
|
|
684
|
+
for al, cands in (p.get("aliases") or {}).items():
|
|
685
|
+
hit = next((suffix[c] for c in cands if c in suffix), None)
|
|
686
|
+
if hit and hit in modules:
|
|
687
|
+
alias_mod.setdefault(p["id"], {})[al] = hit
|
|
688
|
+
cls_names = {c["name"]: k for k, c in classes.items()}
|
|
689
|
+
for fid, f in functions.items():
|
|
690
|
+
mod = f["module"]
|
|
691
|
+
near = set(modules.get(mod, {}).get("deps", [])) | {mod}
|
|
692
|
+
out = set()
|
|
693
|
+
for c in f["calls"]:
|
|
694
|
+
head, _, nm = c.rpartition(".")
|
|
695
|
+
cands = by_name.get(nm, [])
|
|
696
|
+
if not cands:
|
|
697
|
+
continue
|
|
698
|
+
fn = lambda x: functions[x] # noqa: E731
|
|
699
|
+
if head == "self" and f.get("cls"): # self.method(): the same class
|
|
700
|
+
pick = [x for x in cands if fn(x)["module"] == mod and fn(x).get("cls") == f["cls"]]
|
|
701
|
+
elif head: # obj.f(): a module's function, or a method in reach
|
|
702
|
+
aliased = alias_mod.get(mod, {}).get(head)
|
|
703
|
+
mods = [aliased] if aliased else [m for m in mod_tail.get(head, []) if m in near]
|
|
704
|
+
pick = [x for x in cands if fn(x)["module"] in mods and not fn(x).get("cls")]
|
|
705
|
+
if not pick and head not in cls_names:
|
|
706
|
+
meth = [x for x in cands if fn(x).get("cls") and fn(x)["module"] in near]
|
|
707
|
+
pick = meth if len(meth) == 1 else []
|
|
708
|
+
elif not pick:
|
|
709
|
+
pick = [x for x in cands if fn(x).get("cls") == head]
|
|
710
|
+
else: # f(): same module first, then imported modules
|
|
711
|
+
pick = [x for x in cands if fn(x)["module"] == mod and not fn(x).get("cls")] \
|
|
712
|
+
or [x for x in cands if fn(x)["module"] in near and not fn(x).get("cls")]
|
|
713
|
+
for x in pick[:2]:
|
|
714
|
+
if x != fid or nm == f["name"]:
|
|
715
|
+
out.add(x)
|
|
716
|
+
f["callees"] = sorted(out)
|
|
717
|
+
for fid, f in functions.items():
|
|
718
|
+
for x in f["callees"]:
|
|
719
|
+
if x != fid:
|
|
720
|
+
functions[x]["callers"].append(fid)
|
|
721
|
+
routes = _rjs.drop_shadowed(routes)
|
|
722
|
+
for r in routes:
|
|
723
|
+
h = r.get("handler")
|
|
724
|
+
if h:
|
|
725
|
+
r["fid"] = next((fid for fid in (f'{r["module"]}:{h}',) if fid in functions), None) \
|
|
726
|
+
or next((x for x in by_name.get(h.split(".")[-1], []) if functions[x]["module"] in set(modules.get(r["module"], {}).get("deps", [])) | {r["module"]}), None)
|
|
727
|
+
# ---- which functions touch which tables (by ORM class name or SQL text)
|
|
728
|
+
tlow = {t.lower(): t for t in tables}
|
|
729
|
+
cls_table = {c["name"]: c["table"] for c in classes.values() if c.get("table") in tables}
|
|
730
|
+
for fid, f in functions.items():
|
|
731
|
+
touched = set()
|
|
732
|
+
for c in list(f["calls"]) + list(f.get("refs", [])):
|
|
733
|
+
for part in c.split("."):
|
|
734
|
+
if cls_table.get(part):
|
|
735
|
+
touched.add(cls_table[part])
|
|
736
|
+
for t in f.get("sql_refs", []):
|
|
737
|
+
if t.lower() in tlow:
|
|
738
|
+
touched.add(tlow[t.lower()])
|
|
739
|
+
touched |= {k for k in f.get("fs_refs", []) if k in tables}
|
|
740
|
+
f["tables"] = sorted(touched)
|
|
741
|
+
return {"modules": modules, "classes": classes, "tables": tables, "functions": functions, "routes": routes}
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def _sig(kind: str, node: dict) -> str:
|
|
745
|
+
if kind == "modules":
|
|
746
|
+
return node["sha"]
|
|
747
|
+
keys = {"classes": ("bases", "attrs", "methods"), "tables": ("columns", "ops"),
|
|
748
|
+
"functions": ("args", "calls", "complexity", "loc", "decorators")}[kind]
|
|
749
|
+
return json.dumps([node.get(k) for k in keys], sort_keys=True)
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def diff_models(prev: dict | None, cur: dict) -> dict:
|
|
753
|
+
"""node id -> 'new' | 'changed' | 'removed' for each node kind, relative to an earlier step."""
|
|
754
|
+
marks = {"modules": {}, "classes": {}, "tables": {}, "functions": {}}
|
|
755
|
+
if prev is None:
|
|
756
|
+
return marks
|
|
757
|
+
for kind in marks:
|
|
758
|
+
a, b = prev.get(kind, {}), cur.get(kind, {})
|
|
759
|
+
for k in b:
|
|
760
|
+
if k not in a:
|
|
761
|
+
marks[kind][k] = "new"
|
|
762
|
+
elif _sig(kind, a[k]) != _sig(kind, b[k]):
|
|
763
|
+
marks[kind][k] = "changed"
|
|
764
|
+
for k in a:
|
|
765
|
+
if k not in b:
|
|
766
|
+
marks[kind][k] = "removed"
|
|
767
|
+
return marks
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def find_symbols(model: dict, query: str, limit: int = 20) -> list[dict]:
|
|
771
|
+
"""Symbols whose name matches: exact name first, then prefix, then substring. No source code."""
|
|
772
|
+
q = query.strip().lower()
|
|
773
|
+
if not q:
|
|
774
|
+
return []
|
|
775
|
+
rows = []
|
|
776
|
+
for kind in ("functions", "classes", "tables", "modules"):
|
|
777
|
+
for key, n in model.get(kind, {}).items():
|
|
778
|
+
name = (n.get("qual") or n.get("name") or key).lower() if kind != "modules" else key.lower()
|
|
779
|
+
short = name.split(".")[-1].split(":")[-1]
|
|
780
|
+
score = 0 if q in (name, short) else 1 if short.startswith(q) or name.startswith(q) else 2 if q in name or q in key.lower() else None
|
|
781
|
+
if score is not None:
|
|
782
|
+
rows.append((score, kind, key, n))
|
|
783
|
+
rows.sort(key=lambda r: (r[0], len(r[2])))
|
|
784
|
+
return [{"kind": k[:-1] if k != "classes" else "class", "id": key, "file": n.get("file"), "line": n.get("line")} for _, k, key, n in rows[:limit]]
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def resolve(model: dict, ref: str) -> tuple[str, str, dict] | None:
|
|
788
|
+
"""A symbol reference -> (kind, id, node). Accepts a full id (module:qual), a qualified name
|
|
789
|
+
(Class.method), a plain name if it is unique, a table name, a module id or a file path."""
|
|
790
|
+
ref = (ref or "").strip()
|
|
791
|
+
if not ref:
|
|
792
|
+
return None
|
|
793
|
+
for kind in ("functions", "classes", "tables", "modules"):
|
|
794
|
+
if ref in model.get(kind, {}):
|
|
795
|
+
return kind, ref, model[kind][ref]
|
|
796
|
+
for kind, nodes in (("modules", model.get("modules", {})),):
|
|
797
|
+
for k, n in nodes.items():
|
|
798
|
+
if n["file"] == ref:
|
|
799
|
+
return kind, k, n
|
|
800
|
+
hits = []
|
|
801
|
+
for kind in ("functions", "classes"):
|
|
802
|
+
for k, n in model.get(kind, {}).items():
|
|
803
|
+
qual = n.get("qual") or n.get("name")
|
|
804
|
+
if ref in (qual, n.get("name")) or k.endswith(":" + ref) or ref.endswith("." + (qual or "")) and "." in ref:
|
|
805
|
+
hits.append((kind, k, n))
|
|
806
|
+
if len(hits) == 1:
|
|
807
|
+
return hits[0]
|
|
808
|
+
exact = [h for h in hits if (h[2].get("qual") or h[2].get("name")) == ref]
|
|
809
|
+
return exact[0] if len(exact) == 1 else None
|