leyline-code 0.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leyline/__init__.py +3 -0
- leyline/adapters/__init__.py +19 -0
- leyline/adapters/csharp.py +1026 -0
- leyline/adapters/generic.py +844 -0
- leyline/adapters/python.py +929 -0
- leyline/adapters/typescript.py +1512 -0
- leyline/affected.py +509 -0
- leyline/agent_skills.py +311 -0
- leyline/change.py +500 -0
- leyline/channels.py +1782 -0
- leyline/cli.py +1029 -0
- leyline/cluster.py +149 -0
- leyline/context.py +531 -0
- leyline/coupling.py +485 -0
- leyline/coverage.py +376 -0
- leyline/diagrams.py +680 -0
- leyline/diff.py +1078 -0
- leyline/drift.py +747 -0
- leyline/exact.py +410 -0
- leyline/explain.py +973 -0
- leyline/export.py +673 -0
- leyline/fncoupling.py +328 -0
- leyline/formats.py +923 -0
- leyline/grade.py +207 -0
- leyline/incremental.py +931 -0
- leyline/indexer.py +3733 -0
- leyline/learnings.py +743 -0
- leyline/loop.py +584 -0
- leyline/model.py +142 -0
- leyline/outline.py +1066 -0
- leyline/packing.py +86 -0
- leyline/patterns.py +478 -0
- leyline/pr.py +1249 -0
- leyline/props.py +255 -0
- leyline/query.py +683 -0
- leyline/quick.py +853 -0
- leyline/related.py +266 -0
- leyline/removal.py +231 -0
- leyline/renames.py +231 -0
- leyline/rereview.py +375 -0
- leyline/roslyn_refs/Program.cs +179 -0
- leyline/roslyn_refs/RoslynRefs.csproj +16 -0
- leyline/rules.py +171 -0
- leyline/schema.sql +178 -0
- leyline/scip_pb2.py +97 -0
- leyline/server.py +1230 -0
- leyline/skills/leyline-adversarial-review/SKILL.md +121 -0
- leyline/skills/leyline-ask/SKILL.md +57 -0
- leyline/skills/leyline-change-impact/SKILL.md +79 -0
- leyline/skills/leyline-explain-flow/SKILL.md +70 -0
- leyline/skills/leyline-explore-module/SKILL.md +55 -0
- leyline/skills/leyline-pr-review/SKILL.md +77 -0
- leyline/skills/leyline-quick-change/SKILL.md +63 -0
- leyline/skills/leyline-spec/SKILL.md +179 -0
- leyline/skills/leyline-tour/SKILL.md +46 -0
- leyline/spec.py +1819 -0
- leyline/store.py +560 -0
- leyline/tours.py +352 -0
- leyline/verdicts.py +390 -0
- leyline/viewer/viewer.html +1622 -0
- leyline_code-0.8.1.dist-info/METADATA +1325 -0
- leyline_code-0.8.1.dist-info/RECORD +65 -0
- leyline_code-0.8.1.dist-info/WHEEL +4 -0
- leyline_code-0.8.1.dist-info/entry_points.txt +2 -0
- leyline_code-0.8.1.dist-info/licenses/LICENSE +674 -0
leyline/indexer.py
ADDED
|
@@ -0,0 +1,3733 @@
|
|
|
1
|
+
"""Walk a repo, run the language adapters, resolve names across files, and write facts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import gc
|
|
6
|
+
import hashlib
|
|
7
|
+
import itertools
|
|
8
|
+
import posixpath
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import pickle
|
|
12
|
+
import re
|
|
13
|
+
import subprocess
|
|
14
|
+
import time
|
|
15
|
+
import sys
|
|
16
|
+
from array import array
|
|
17
|
+
from collections import Counter, defaultdict
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
from . import packing, store
|
|
22
|
+
from .adapters import BY_EXTENSION, BY_LANGUAGE
|
|
23
|
+
from .adapters.python import module_path
|
|
24
|
+
from .model import CallSite, Edge, Endpoint, FieldUse, FileResult, Node
|
|
25
|
+
|
|
26
|
+
SOURCE = "leyline-indexer/0.1"
|
|
27
|
+
MODULE_MARKERS = ("pyproject.toml", "setup.py", "package.json", "__init__.py")
|
|
28
|
+
# Languages where a file is a module and other files name what they take from it (import { a } from "./b").
|
|
29
|
+
FILE_MODULE = ("python", "typescript")
|
|
30
|
+
CTORS = (".ctor", "__init__", "constructor")
|
|
31
|
+
NODE_BUILTINS = frozenset("""assert async_hooks buffer child_process cluster console constants crypto dgram dns domain
|
|
32
|
+
events fs http http2 https inspector module net os path perf_hooks process punycode querystring readline repl stream
|
|
33
|
+
string_decoder timers tls tty url util v8 vm worker_threads zlib test""".split())
|
|
34
|
+
SKIP_DIRS = {".git", "node_modules", "bin", "obj", "__pycache__", ".venv", "venv", ".godot", ".leyline"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# Directories of other people's code, left out even when committed: mapped, they bury the repository's own code.
|
|
38
|
+
DEPENDENCY_DIRS = {"node_modules", "bower_components", "site-packages", "__pycache__"}
|
|
39
|
+
# A vendor directory is only someone else's code when a package manager says so (Go modules, Composer).
|
|
40
|
+
VENDOR_MARKERS = ("vendor/modules.txt", "vendor/autoload.php")
|
|
41
|
+
MAX_FILE_MB = 5 # a source file larger than this is generated; parsing one costs seconds and hundreds of MB
|
|
42
|
+
LONG_LINES = 2000 # average characters per line above which a file is minified or generated, not written
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _git(root: Path, *args: str, raw: bool = False, timeout: float = 30):
|
|
46
|
+
try:
|
|
47
|
+
out = subprocess.run(["git", "-C", str(root), *args], capture_output=True, timeout=timeout,
|
|
48
|
+
stdin=subprocess.DEVNULL)
|
|
49
|
+
except (OSError, subprocess.SubprocessError):
|
|
50
|
+
return None
|
|
51
|
+
if out.returncode != 0:
|
|
52
|
+
return None
|
|
53
|
+
# git writes paths and names as UTF-8 bytes whatever the platform's locale is
|
|
54
|
+
return out.stdout if raw else out.stdout.decode("utf-8", "replace").strip()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def max_file_bytes() -> int:
|
|
58
|
+
env = os.environ.get("LEYLINE_MAX_FILE_MB", "")
|
|
59
|
+
try:
|
|
60
|
+
return int(float(env) * 2**20) if env else MAX_FILE_MB * 2**20
|
|
61
|
+
except ValueError:
|
|
62
|
+
return MAX_FILE_MB * 2**20
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Listing:
|
|
66
|
+
"""The files of a repository that can be read, and the ones left out with the reason for each. `how` is git or
|
|
67
|
+
walk; `why_walk` says why git did not list them, when it is a git repository."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, files: list[str], skipped: list[tuple[str, str]], how: str, why_walk: Optional[str] = None):
|
|
70
|
+
self.files, self.skipped, self.how, self.why_walk = files, skipped, how, why_walk
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def scan(root: Path, names: Optional[list[str]] = None) -> Listing:
|
|
74
|
+
"""Every file to index: what git lists (tracked, and untracked but not ignored), or a walk of the directory when
|
|
75
|
+
it is not a git repository. Anything that cannot be read as a source file is left out, with a reason, so that
|
|
76
|
+
nothing later fails on it: entries git lists that are not files on disk (submodules, deleted files, symlinks to
|
|
77
|
+
directories or to nothing), files outside the repository, other people's code, and source files that are too
|
|
78
|
+
large, binary, minified or unreadable. `names` lists the files instead, as git would (a commit's files written
|
|
79
|
+
out by `leyline pr`)."""
|
|
80
|
+
root = Path(root)
|
|
81
|
+
skipped: list[tuple[str, str]] = []
|
|
82
|
+
why_walk = None
|
|
83
|
+
out = None if names is not None else \
|
|
84
|
+
_git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard", raw=True, timeout=600)
|
|
85
|
+
if names is not None:
|
|
86
|
+
how, given, names = "git", names, []
|
|
87
|
+
for n in given:
|
|
88
|
+
try:
|
|
89
|
+
n.encode("utf-8")
|
|
90
|
+
names.append(n)
|
|
91
|
+
except UnicodeEncodeError:
|
|
92
|
+
skipped.append((n.encode("utf-8", "surrogateescape").decode("utf-8", "replace"), "file name is not UTF-8"))
|
|
93
|
+
names = sorted(set(names))
|
|
94
|
+
elif out is not None:
|
|
95
|
+
names, how = [], "git"
|
|
96
|
+
for b in out.split(b"\0"):
|
|
97
|
+
if not b:
|
|
98
|
+
continue
|
|
99
|
+
try:
|
|
100
|
+
names.append(b.decode("utf-8"))
|
|
101
|
+
except UnicodeDecodeError:
|
|
102
|
+
skipped.append((b.decode("utf-8", "replace"), "file name is not UTF-8"))
|
|
103
|
+
names = sorted(set(n.rstrip("/") for n in names)) # an untracked nested repository is listed as "dir/"
|
|
104
|
+
else:
|
|
105
|
+
names, how = _walk(root, skipped), "walk"
|
|
106
|
+
if (root / ".git").exists():
|
|
107
|
+
try:
|
|
108
|
+
run = subprocess.run(["git", "-C", str(root), "ls-files", "-z"], capture_output=True, timeout=600,
|
|
109
|
+
stdin=subprocess.DEVNULL)
|
|
110
|
+
why_walk = run.stderr.decode("utf-8", "replace").strip().splitlines()
|
|
111
|
+
why_walk = why_walk[0] if why_walk else f"git exited with {run.returncode}"
|
|
112
|
+
except (OSError, subprocess.SubprocessError) as e:
|
|
113
|
+
why_walk = f"git could not be run ({e})"
|
|
114
|
+
vendored = {m[: -len(m.rsplit("/", 1)[-1])] for m in names
|
|
115
|
+
if any(m == v or m.endswith("/" + v) for v in VENDOR_MARKERS)}
|
|
116
|
+
try:
|
|
117
|
+
real_root = root.resolve()
|
|
118
|
+
except OSError:
|
|
119
|
+
real_root = root
|
|
120
|
+
limit = max_file_bytes()
|
|
121
|
+
files, listed = [], set(names)
|
|
122
|
+
for f in names:
|
|
123
|
+
parts = f.split("/")
|
|
124
|
+
if set(parts[:-1]) & DEPENDENCY_DIRS or any(f.startswith(v) for v in vendored):
|
|
125
|
+
skipped.append((f, "dependency directory"))
|
|
126
|
+
continue
|
|
127
|
+
why = _unusable(root, real_root, f, BY_EXTENSION, limit, listed)
|
|
128
|
+
if why:
|
|
129
|
+
skipped.append((f, why))
|
|
130
|
+
else:
|
|
131
|
+
files.append(f)
|
|
132
|
+
return Listing(files, skipped, how, why_walk)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _walk(root: Path, skipped: list) -> list[str]:
|
|
136
|
+
"""The files under root when git cannot list them. Symlinked directories are not followed (a link back up the
|
|
137
|
+
tree would never end), nested repositories and build or tool directories are left out, and the ignore files git
|
|
138
|
+
would apply are: the person's global excludes file, .git/info/exclude, then each .gitignore (see _ignore_rules).
|
|
139
|
+
git may fail on a real repository (a broken .git, a dubious-ownership refusal), and a file kept out of it with
|
|
140
|
+
one of those (.env) must stay out of the map and its page here too."""
|
|
141
|
+
files = []
|
|
142
|
+
top = [r for f in (_global_excludes(), root / ".git" / "info" / "exclude") if f is not None
|
|
143
|
+
for r in _ignore_rules(f, "")]
|
|
144
|
+
|
|
145
|
+
def err(e: OSError) -> None:
|
|
146
|
+
try:
|
|
147
|
+
rel = Path(e.filename).relative_to(root).as_posix() if e.filename else "?"
|
|
148
|
+
except ValueError:
|
|
149
|
+
rel = str(e.filename)
|
|
150
|
+
skipped.append((rel, f"cannot read directory: {e.strerror or e}"))
|
|
151
|
+
rules: dict[str, list] = {} # directory -> the ignore rules in force there, its own last
|
|
152
|
+
for d, dirs, names in os.walk(root, onerror=err):
|
|
153
|
+
rel = Path(d).relative_to(root).as_posix()
|
|
154
|
+
rel = "" if rel == "." else rel + "/"
|
|
155
|
+
here = rules.get(rel[:-1].rpartition("/")[0] + "/" if "/" in rel[:-1] else "", []) if rel else top
|
|
156
|
+
if ".gitignore" in names:
|
|
157
|
+
here = here + _ignore_rules(Path(d) / ".gitignore", rel)
|
|
158
|
+
rules[rel] = here
|
|
159
|
+
keep = []
|
|
160
|
+
for x in dirs:
|
|
161
|
+
p = os.path.join(d, x)
|
|
162
|
+
if x in SKIP_DIRS or x in DEPENDENCY_DIRS or _ignored(here, rel + x, True):
|
|
163
|
+
continue
|
|
164
|
+
if os.path.islink(p):
|
|
165
|
+
skipped.append((rel + x, "symlink to a directory"))
|
|
166
|
+
elif os.path.exists(os.path.join(p, ".git")):
|
|
167
|
+
skipped.append((rel + x, "nested repository (map it on its own, or with this one as a workspace)"))
|
|
168
|
+
else:
|
|
169
|
+
keep.append(x)
|
|
170
|
+
dirs[:] = sorted(keep)
|
|
171
|
+
for x in names:
|
|
172
|
+
try:
|
|
173
|
+
(rel + x).encode("utf-8")
|
|
174
|
+
except UnicodeEncodeError:
|
|
175
|
+
skipped.append(((rel + x).encode("utf-8", "surrogateescape").decode("utf-8", "replace"),
|
|
176
|
+
"file name is not UTF-8"))
|
|
177
|
+
continue
|
|
178
|
+
if not _ignored(here, rel + x, False):
|
|
179
|
+
files.append(rel + x)
|
|
180
|
+
return sorted(files)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _global_excludes() -> Optional[Path]:
|
|
184
|
+
"""The person's global ignore file: git's core.excludesFile, else $XDG_CONFIG_HOME/git/ignore (~/.config/git/
|
|
185
|
+
ignore). Asked of git outside any repository, so a repository's own config has no say."""
|
|
186
|
+
try:
|
|
187
|
+
run = subprocess.run(["git", "config", "--get", "core.excludesFile"], capture_output=True, timeout=30,
|
|
188
|
+
stdin=subprocess.DEVNULL, cwd=str(Path.home()))
|
|
189
|
+
named = run.stdout.decode("utf-8", "replace").strip() if run.returncode == 0 else ""
|
|
190
|
+
except (OSError, subprocess.SubprocessError, RuntimeError):
|
|
191
|
+
named = ""
|
|
192
|
+
if named:
|
|
193
|
+
return Path(os.path.expanduser(named))
|
|
194
|
+
xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
|
|
195
|
+
return Path(xdg) / "git" / "ignore"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _ignore_rules(path: Path, base: str) -> list[tuple]:
|
|
199
|
+
"""An ignore file's patterns as (regex, negated, directories only), read as git reads them: names, globs with
|
|
200
|
+
* ? [..] and **, a leading / or an inner / to anchor, a trailing / for directories, ! to take a path back, a
|
|
201
|
+
backslash before a character to mean it as written (\\#, \\!, \\*, a trailing \\ ), and trailing spaces cut."""
|
|
202
|
+
import re as _re
|
|
203
|
+
try:
|
|
204
|
+
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
205
|
+
except OSError:
|
|
206
|
+
return []
|
|
207
|
+
out = []
|
|
208
|
+
for line in lines:
|
|
209
|
+
while line.endswith(" ") and not line.endswith("\\ "): # trailing spaces go; git keeps an escaped one
|
|
210
|
+
line = line[:-1]
|
|
211
|
+
if not line or line.startswith("#"):
|
|
212
|
+
continue
|
|
213
|
+
neg = line.startswith("!")
|
|
214
|
+
if neg:
|
|
215
|
+
line = line[1:]
|
|
216
|
+
dir_only = line.endswith("/")
|
|
217
|
+
line = line.rstrip("/") if dir_only else line
|
|
218
|
+
anchored = "/" in line # a leading or inner slash; the trailing one only says "a directory"
|
|
219
|
+
line = line.lstrip("/")
|
|
220
|
+
if not line:
|
|
221
|
+
continue
|
|
222
|
+
rx, i = "", 0
|
|
223
|
+
while i < len(line):
|
|
224
|
+
c = line[i]
|
|
225
|
+
if c == "\\" and i + 1 < len(line): # the next character as written
|
|
226
|
+
rx, i = rx + _re.escape(line[i + 1]), i + 2
|
|
227
|
+
elif line.startswith("**/", i):
|
|
228
|
+
rx, i = rx + "(?:.*/)?", i + 3
|
|
229
|
+
elif line.startswith("/**", i) and i + 3 == len(line):
|
|
230
|
+
rx, i = rx + "/.*", i + 3
|
|
231
|
+
elif line.startswith("**", i):
|
|
232
|
+
rx, i = rx + ".*", i + 2
|
|
233
|
+
elif c == "*":
|
|
234
|
+
rx, i = rx + "[^/]*", i + 1
|
|
235
|
+
elif c == "?":
|
|
236
|
+
rx, i = rx + "[^/]", i + 1
|
|
237
|
+
elif c == "[" and "]" in line[i + 1:]:
|
|
238
|
+
j = line.index("]", i + 1)
|
|
239
|
+
body = line[i + 1:j]
|
|
240
|
+
rx, i = rx + "[" + ("^" + body[1:] if body.startswith("!") else body) + "]", j + 1
|
|
241
|
+
else:
|
|
242
|
+
rx, i = rx + _re.escape(c), i + 1
|
|
243
|
+
prefix = _re.escape(base) if anchored else _re.escape(base) + "(?:.*/)?"
|
|
244
|
+
out.append((_re.compile(prefix + rx + "$"), neg, dir_only))
|
|
245
|
+
return out
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _ignored(rules: list, path: str, is_dir: bool) -> bool:
|
|
249
|
+
hit = False
|
|
250
|
+
for rx, neg, dir_only in rules: # the last rule that matches decides, as in git
|
|
251
|
+
if (is_dir or not dir_only) and rx.match(path):
|
|
252
|
+
hit = not neg
|
|
253
|
+
return hit
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _unusable(root: Path, real_root: Path, f: str, sources: dict, limit: int, listed: set) -> Optional[str]:
|
|
257
|
+
"""Why a listed path cannot be indexed, or None. Only source files (those an adapter reads) are opened."""
|
|
258
|
+
import errno
|
|
259
|
+
import stat as st
|
|
260
|
+
p = root / f
|
|
261
|
+
try:
|
|
262
|
+
info = os.lstat(p)
|
|
263
|
+
except FileNotFoundError:
|
|
264
|
+
return "deleted (git still lists it)"
|
|
265
|
+
except OSError as e:
|
|
266
|
+
return f"cannot read: {e.strerror or e}"
|
|
267
|
+
except ValueError: # UnicodeEncodeError: a name the file system's encoding (the locale's) cannot spell
|
|
268
|
+
return "file name cannot be spelled in this system's encoding (use a UTF-8 locale)"
|
|
269
|
+
if st.S_ISLNK(info.st_mode):
|
|
270
|
+
try:
|
|
271
|
+
target = p.resolve(strict=True)
|
|
272
|
+
info = os.stat(p)
|
|
273
|
+
except FileNotFoundError:
|
|
274
|
+
return "broken symlink"
|
|
275
|
+
except RuntimeError: # a symlink loop, on Python before 3.13
|
|
276
|
+
return "symlink loop"
|
|
277
|
+
except OSError as e:
|
|
278
|
+
return "symlink loop" if e.errno == errno.ELOOP else f"cannot read: {e.strerror or e}"
|
|
279
|
+
if st.S_ISDIR(info.st_mode):
|
|
280
|
+
return "symlink to a directory"
|
|
281
|
+
try:
|
|
282
|
+
inside = target.relative_to(real_root).as_posix()
|
|
283
|
+
except ValueError:
|
|
284
|
+
return "symlink to outside the repository"
|
|
285
|
+
if inside in listed:
|
|
286
|
+
return "symlink to another listed file (indexed there)"
|
|
287
|
+
if st.S_ISDIR(info.st_mode):
|
|
288
|
+
return "submodule or nested repository (map it on its own, or with this one as a workspace)"
|
|
289
|
+
if not st.S_ISREG(info.st_mode):
|
|
290
|
+
return "not a regular file"
|
|
291
|
+
ext = "." + f.rsplit(".", 1)[-1] if "." in f.rsplit("/", 1)[-1] else ""
|
|
292
|
+
if ext not in sources:
|
|
293
|
+
return None
|
|
294
|
+
if info.st_size > limit:
|
|
295
|
+
return f"larger than {limit / 2**20:g} MB (generated?)"
|
|
296
|
+
if f.endswith((".min.js", ".min.mjs", ".min.cjs")):
|
|
297
|
+
return "minified"
|
|
298
|
+
try:
|
|
299
|
+
with open(p, "rb") as fh:
|
|
300
|
+
head = fh.read(65536)
|
|
301
|
+
except OSError as e:
|
|
302
|
+
return f"cannot read: {e.strerror or e}"
|
|
303
|
+
if _binary(head[:8192]):
|
|
304
|
+
return "binary"
|
|
305
|
+
# a bundle of a few tens of KB on one line is as minified as a larger one; a short file is let through
|
|
306
|
+
if len(head) >= 4 * LONG_LINES and head.count(b"\n") < len(head) // LONG_LINES:
|
|
307
|
+
return "minified or generated (very long lines)"
|
|
308
|
+
return None
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
_CONTROL = bytes(b for b in range(32) if b not in (9, 10, 12, 13, 27)) + b"\x7f"
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _binary(head: bytes) -> bool:
|
|
315
|
+
"""Whether the start of a file is data rather than text. A NUL byte alone does not decide it (source can hold
|
|
316
|
+
one inside a string literal); one byte in twenty being a control character does. UTF-16 text is half NULs, so a
|
|
317
|
+
byte order mark lets it through."""
|
|
318
|
+
if not head or head.startswith((b"\xff\xfe", b"\xfe\xff")):
|
|
319
|
+
return False
|
|
320
|
+
return len(head) - len(head.translate(None, _CONTROL)) > len(head) // 20
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def list_files(root: Path) -> list[str]:
|
|
324
|
+
return scan(root).files
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _report_skipped(repo: str, skipped: list) -> None:
|
|
328
|
+
"""One line per reason on stderr, with a few of the files, so a person can see what the map leaves out."""
|
|
329
|
+
quiet = ("dependency directory", "deleted (git still lists it)", "symlink to another listed file (indexed there)")
|
|
330
|
+
by = defaultdict(list)
|
|
331
|
+
for f, why in skipped:
|
|
332
|
+
by[why].append(f)
|
|
333
|
+
for why, fs in sorted(by.items(), key=lambda kv: -len(kv[1])):
|
|
334
|
+
if why in quiet:
|
|
335
|
+
line = f"leyline: {repo}: left out {len(fs)} file{'s' * (len(fs) != 1)}: {why}"
|
|
336
|
+
else:
|
|
337
|
+
shown = ", ".join(fs[:3]) + (f" and {len(fs) - 3} more" if len(fs) > 3 else "")
|
|
338
|
+
line = f"leyline: {repo}: left out {why}: {shown}"
|
|
339
|
+
if REPORTED is None or line not in REPORTED: # a command that maps twice (a pull request's base and head)
|
|
340
|
+
if REPORTED is not None: # says it once
|
|
341
|
+
REPORTED.add(line)
|
|
342
|
+
print(line, file=sys.stderr)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
REPORTED: Optional[set] = None # the lines said so far by the command running (leyline.cli sets it)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def skipped_summary(skipped: list, failed: list) -> dict:
|
|
349
|
+
"""What a run left out and why, for the stats and the store: reason -> count and a few examples."""
|
|
350
|
+
out: dict = {}
|
|
351
|
+
for f, why in list(skipped) + [(f, "failed to parse: " + (w or "")) for f, w in failed]:
|
|
352
|
+
key = why if not why.startswith("failed to parse") else "failed to parse"
|
|
353
|
+
got = out.setdefault(key, {"count": 0, "examples": []})
|
|
354
|
+
got["count"] += 1
|
|
355
|
+
if len(got["examples"]) < 5:
|
|
356
|
+
got["examples"].append(f if key != "failed to parse" else f"{f}: {why[len('failed to parse: '):]}")
|
|
357
|
+
return out
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# The placeholders a table-driven test's title is filled in from (it.each, test.each, pytest ids): printf-style
|
|
361
|
+
# %s %d %j ..., $name and ${name} from a table, and Jest's %# for the row number.
|
|
362
|
+
_TITLE_SLOT = re.compile(r"%[sdifjoOpc#]|\$\{?[A-Za-z_][\w.]*\}?")
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _test_title(n) -> str:
|
|
366
|
+
"""A test's name as a flow shows it. A title that is a template (`%s/%s` from it.each) says little on its own,
|
|
367
|
+
so it is shown after its suite, with each placeholder as an ellipsis."""
|
|
368
|
+
title = n.name
|
|
369
|
+
if not _TITLE_SLOT.search(title):
|
|
370
|
+
return title
|
|
371
|
+
shown = _TITLE_SLOT.sub("…", title.replace("%%", "%")).strip()
|
|
372
|
+
suite = _TITLE_SLOT.sub("…", str(n.attrs.get("suite") or "")).strip()
|
|
373
|
+
return f"{suite} > {shown}" if suite else shown
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def read_source(path: Path) -> bytes:
|
|
377
|
+
"""A source file's bytes as the parsers take them: UTF-8. A UTF-16 file (it starts with a byte order mark,
|
|
378
|
+
as some Windows editors write C#) is converted; lines stay where they were."""
|
|
379
|
+
data = Path(path).read_bytes()
|
|
380
|
+
if data.startswith((b"\xff\xfe", b"\xfe\xff")):
|
|
381
|
+
try:
|
|
382
|
+
return data.decode("utf-16").encode("utf-8")
|
|
383
|
+
except UnicodeDecodeError:
|
|
384
|
+
return data
|
|
385
|
+
if str(path).endswith((".py", ".pyi", ".pyw")): # a PEP 263 declaration in the first two lines: # coding: latin-1
|
|
386
|
+
m = _CODING.match(b"\n".join(data.split(b"\n", 2)[:2]))
|
|
387
|
+
if m and codecs_lookup(m[1].decode("ascii")) not in (None, "utf-8"):
|
|
388
|
+
try:
|
|
389
|
+
return data.decode(m[1].decode("ascii")).encode("utf-8")
|
|
390
|
+
except (UnicodeDecodeError, LookupError):
|
|
391
|
+
return data
|
|
392
|
+
return data
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _jsonc(path: Path) -> Optional[dict]:
|
|
396
|
+
"""A JSON file that may hold comments and trailing commas, as tsconfig.json does; None when it cannot be read."""
|
|
397
|
+
try:
|
|
398
|
+
text = path.read_text(encoding="utf-8-sig", errors="replace")
|
|
399
|
+
except OSError:
|
|
400
|
+
return None
|
|
401
|
+
# comments outside strings go, then commas before a closing bracket
|
|
402
|
+
text = re.sub(r'("(?:\\.|[^"\\])*")|//[^\n]*|/\*.*?\*/', lambda m: m[1] or "", text, flags=re.S)
|
|
403
|
+
text = re.sub(r'("(?:\\.|[^"\\])*")|,(\s*[}\]])', lambda m: m[1] or m[2], text)
|
|
404
|
+
try:
|
|
405
|
+
data = json.loads(text)
|
|
406
|
+
except (ValueError, RecursionError): # nested past the recursion limit: Python before 3.13 raises this
|
|
407
|
+
return None
|
|
408
|
+
return data if isinstance(data, dict) else None
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
_CODING = re.compile(rb"(?:[ \t\f]*(?:#[^\n]*)?\n)?[ \t\f]*#[^\n]*?coding[:=][ \t]*([-\w.]+)")
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def codecs_lookup(name: str) -> Optional[str]:
|
|
415
|
+
"""A codec's canonical name, or None when Python has no such codec."""
|
|
416
|
+
import codecs
|
|
417
|
+
try:
|
|
418
|
+
return codecs.lookup(name).name
|
|
419
|
+
except LookupError:
|
|
420
|
+
return None
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def source_lines(path: Path) -> list[str]:
|
|
424
|
+
"""A source file's lines, numbered as the parsers number them: only a newline ends a line (str.splitlines also
|
|
425
|
+
breaks at form feeds and other separators, which put every later line out of step with the syntax tree)."""
|
|
426
|
+
return text_lines(read_source(path))
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def text_lines(data: bytes) -> list[str]:
|
|
430
|
+
return [ln[:-1] if ln.endswith("\r") else ln for ln in data.decode("utf-8", "replace").split("\n")]
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _module_dirs(files: list[str]) -> set[str]:
|
|
434
|
+
"""Directories that are modules because they hold a project marker."""
|
|
435
|
+
dirs, packages = set(), set()
|
|
436
|
+
for f in files:
|
|
437
|
+
d, _, base = f.rpartition("/")
|
|
438
|
+
if base.endswith(".csproj") or base in MODULE_MARKERS:
|
|
439
|
+
dirs.add(d)
|
|
440
|
+
if base == "__init__.py":
|
|
441
|
+
packages.add(d)
|
|
442
|
+
# A nested __init__.py belongs to its top-most package, not to a module of its own. A project file
|
|
443
|
+
# inside another project's folder (a workspace package) is a module of its own. Both are looked up by
|
|
444
|
+
# directory: scanning every file per package, and every directory per directory, was quadratic.
|
|
445
|
+
projects = {f.rpartition("/")[0] for f in files
|
|
446
|
+
if f.endswith(".csproj") or f.rsplit("/", 1)[-1] in MODULE_MARKERS[:3]}
|
|
447
|
+
own = {d for d in dirs if d not in packages or d in projects}
|
|
448
|
+
|
|
449
|
+
def nested(d: str) -> bool:
|
|
450
|
+
parts = d.split("/")
|
|
451
|
+
return any("/".join(parts[:k]) in dirs for k in range(1, len(parts)))
|
|
452
|
+
return {d for d in dirs if d in own or not nested(d)}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _module_for(path: str, module_dirs: set[str]) -> str:
|
|
456
|
+
d = path.rpartition("/")[0]
|
|
457
|
+
cur = d
|
|
458
|
+
while True:
|
|
459
|
+
if cur in module_dirs and cur != "":
|
|
460
|
+
return cur
|
|
461
|
+
if "/" not in cur:
|
|
462
|
+
break
|
|
463
|
+
cur = cur.rpartition("/")[0]
|
|
464
|
+
if cur in module_dirs and cur != "":
|
|
465
|
+
return cur
|
|
466
|
+
return d # no marker: the file's own directory ("" is the repo root)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _ext_clashes(work: list) -> set[str]:
|
|
470
|
+
"""The files whose ids keep their extension: files of one language in one directory that differ only in their
|
|
471
|
+
extension (a.js beside a.ts, m.c beside m.h) would name their contents alike. The first by path keeps the usual
|
|
472
|
+
ids, so a directory with no such pair keeps them all; the others keep the extension in theirs."""
|
|
473
|
+
import inspect
|
|
474
|
+
groups: dict[tuple, list] = defaultdict(list)
|
|
475
|
+
for w in work:
|
|
476
|
+
adapter = BY_EXTENSION[w[1]]
|
|
477
|
+
if "keep_ext" in inspect.signature(adapter.parse).parameters:
|
|
478
|
+
groups[(adapter.LANGUAGE, adapter.module_path(w[0]))].append(w[0])
|
|
479
|
+
return {f for fs in groups.values() if len(fs) > 1 for f in sorted(fs)[1:]}
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _arity(type_id: str) -> int:
|
|
483
|
+
tail = type_id.rsplit("`", 1)
|
|
484
|
+
return int(tail[1]) if len(tail) == 2 and tail[1].isdigit() else 0
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
NOT_TYPES = ("String", "Self", "None", "Some", "Ok", "Err")
|
|
488
|
+
_IDENT = re.compile(r"[A-Za-z_]\w*")
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _declared(text: list[str], start: Optional[int], end: Optional[int]) -> dict:
|
|
492
|
+
"""Variables a span of text declares with a type: name -> the type names written for it, in the order that
|
|
493
|
+
decides (the first, then any later one), so that which of them are in the repo can be settled later."""
|
|
494
|
+
body = "\n".join(text[(start or 1) - 1:end or start or 1])
|
|
495
|
+
out: dict[str, list] = {}
|
|
496
|
+
for pat, gn, gt in Indexer._DECL:
|
|
497
|
+
for m in pat.finditer(body):
|
|
498
|
+
name, tname = m.group(gn), m.group(gt)
|
|
499
|
+
seen = out.setdefault(name, [])
|
|
500
|
+
if tname not in seen:
|
|
501
|
+
seen.append(tname)
|
|
502
|
+
return out
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _parse_one(job):
|
|
506
|
+
"""Parse one file: (path, ext, module dir, module id, lines, sha1, pickled (result, declarations) or None,
|
|
507
|
+
error or None, the names the result mentions). Runs in a worker process on large repositories, so it touches
|
|
508
|
+
nothing but its arguments. The result comes back pickled because that is how it would cross from the worker
|
|
509
|
+
anyway, and the same bytes are what the parse cache keeps (leyline.incremental); its calls are packed apart
|
|
510
|
+
(leyline.packing)."""
|
|
511
|
+
from .incremental import mentions
|
|
512
|
+
try:
|
|
513
|
+
out = _parse_file(job)
|
|
514
|
+
res, decls = out[6], out[8]
|
|
515
|
+
if res is None:
|
|
516
|
+
return (*out[:6], None, out[7], b"")
|
|
517
|
+
return (*out[:6], packing.pack(res, decls), None, mentions(res, decls))
|
|
518
|
+
except Exception as exc: # MemoryError, or a result that will not pickle: the file fails, not the run
|
|
519
|
+
f, ext, mod_dir, mod_id = job[2]
|
|
520
|
+
return f, ext, mod_dir, mod_id, 1, None, None, f"{type(exc).__name__}: {exc}"[:300], b""
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
DEEP_RECURSION = 25_000 # frames allowed when a file is parsed again on a large stack; past this, the work grows too fast
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _deep(fn, *args):
|
|
527
|
+
"""fn(*args) on a thread with a 1 GB stack and a recursion limit to match. The adapters walk the syntax tree
|
|
528
|
+
recursively; a file nested deeper than the default limit allows (a long else-if chain, a generated expression)
|
|
529
|
+
is read this way instead of being lost. The stack is only reserved address space until it is used."""
|
|
530
|
+
import threading
|
|
531
|
+
out: list = []
|
|
532
|
+
|
|
533
|
+
def run():
|
|
534
|
+
try:
|
|
535
|
+
out.append((True, fn(*args)))
|
|
536
|
+
except BaseException as exc: # handed back to the caller's thread
|
|
537
|
+
out.append((False, exc))
|
|
538
|
+
if os.name == "nt" and sys.version_info < (3, 12):
|
|
539
|
+
# Python before 3.12 on Windows does not stop a deep recursion before the thread's stack runs out: the whole
|
|
540
|
+
# process dies. The file is reported as too deeply nested instead.
|
|
541
|
+
raise RecursionError("nested too deeply to read")
|
|
542
|
+
old_stack, old_limit = threading.stack_size(), sys.getrecursionlimit()
|
|
543
|
+
try:
|
|
544
|
+
threading.stack_size(2**30)
|
|
545
|
+
except (ValueError, RuntimeError): # a platform that will not give a thread a large stack
|
|
546
|
+
raise RecursionError("nested too deeply to read")
|
|
547
|
+
try:
|
|
548
|
+
sys.setrecursionlimit(max(old_limit, DEEP_RECURSION))
|
|
549
|
+
t = threading.Thread(target=run, name="leyline-deep-parse")
|
|
550
|
+
t.start()
|
|
551
|
+
t.join()
|
|
552
|
+
finally:
|
|
553
|
+
threading.stack_size(old_stack)
|
|
554
|
+
sys.setrecursionlimit(old_limit)
|
|
555
|
+
ok, value = out[0]
|
|
556
|
+
if not ok:
|
|
557
|
+
raise value
|
|
558
|
+
return value
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _parse_file(job):
|
|
562
|
+
root, repo, (f, ext, mod_dir, mod_id) = job[:3]
|
|
563
|
+
try:
|
|
564
|
+
data = read_source(Path(root) / f)
|
|
565
|
+
except OSError as exc: # gone or made unreadable since it was listed
|
|
566
|
+
return f, ext, mod_dir, mod_id, 1, None, None, f"cannot read: {exc.strerror or exc}", None
|
|
567
|
+
loc, sha = data.count(b"\n") + 1, hashlib.sha1(data).hexdigest()
|
|
568
|
+
adapter = BY_EXTENSION[ext]
|
|
569
|
+
args = (repo, f, f"{repo}:file:{f}", data, mod_dir or ".")
|
|
570
|
+
parse = adapter.parse
|
|
571
|
+
if len(job) > 3 and job[3]: # see _ext_clashes
|
|
572
|
+
import functools
|
|
573
|
+
parse = functools.partial(adapter.parse, keep_ext=True)
|
|
574
|
+
try:
|
|
575
|
+
try:
|
|
576
|
+
res = parse(*args)
|
|
577
|
+
except RecursionError:
|
|
578
|
+
res = _deep(parse, *args)
|
|
579
|
+
except RecursionError:
|
|
580
|
+
return f, ext, mod_dir, mod_id, loc, sha, None, "nested too deeply to read", None
|
|
581
|
+
except Exception as exc:
|
|
582
|
+
return f, ext, mod_dir, mod_id, loc, sha, None, f"{type(exc).__name__}: {exc}"[:300], None
|
|
583
|
+
lines = data.split(b"\n")
|
|
584
|
+
for n in res.nodes:
|
|
585
|
+
if n.span_start and n.kind in ("type", "callable", "test", "field"):
|
|
586
|
+
# A hash of the node's own text, so a later index can tell which nodes were edited.
|
|
587
|
+
n.content_hash = hashlib.sha1(b"\n".join(ln.strip() for ln in lines[n.span_start - 1:n.span_end])).hexdigest()[:16]
|
|
588
|
+
decls = None
|
|
589
|
+
if getattr(adapter, "GENERIC", False):
|
|
590
|
+
# The generic resolver reads variable types out of function and type text. Reading it here puts that
|
|
591
|
+
# work in the parallel part; the text is read the way Indexer._file_text reads it, so the answer is the same.
|
|
592
|
+
text = text_lines(data)
|
|
593
|
+
if text is not None:
|
|
594
|
+
# Only where a call names a variable receiver is the type looked for; any other place falls back to
|
|
595
|
+
# reading the text in the main process.
|
|
596
|
+
holders, todo = set(), list(res.calls)
|
|
597
|
+
while todo:
|
|
598
|
+
c = todo.pop()
|
|
599
|
+
if c.receiver not in (None, "this", "base", "?") and _IDENT.fullmatch(c.receiver):
|
|
600
|
+
holders.update((c.src_id, c.enclosing_type))
|
|
601
|
+
if c.chain is not None:
|
|
602
|
+
todo.append(c.chain)
|
|
603
|
+
decls = {n.id: (n.span_start, n.span_end, _declared(text, n.span_start, n.span_end))
|
|
604
|
+
for n in res.nodes if n.id in holders}
|
|
605
|
+
return f, ext, mod_dir, mod_id, loc, sha, res, None, decls
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
PARALLEL_MIN_FILES = 300 # below this, starting worker processes costs more than it saves
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _jobs() -> int:
|
|
612
|
+
env = os.environ.get("LEYLINE_JOBS")
|
|
613
|
+
if env and env.isdigit():
|
|
614
|
+
return max(1, int(env))
|
|
615
|
+
return max(1, min(os.cpu_count() or 1, 16))
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def _start_method() -> Optional[str]:
|
|
619
|
+
"""How worker processes start. fork is quickest, but it is only safe in a process with one thread (a fork
|
|
620
|
+
taken while another thread holds a lock deadlocks the child), and macOS and Windows do not have it safely or
|
|
621
|
+
at all; spawn works everywhere. LEYLINE_START_METHOD picks one (fork, forkserver or spawn)."""
|
|
622
|
+
import multiprocessing
|
|
623
|
+
import threading
|
|
624
|
+
env = os.environ.get("LEYLINE_START_METHOD")
|
|
625
|
+
methods = multiprocessing.get_all_start_methods()
|
|
626
|
+
if env:
|
|
627
|
+
if env not in methods:
|
|
628
|
+
raise ValueError(f"LEYLINE_START_METHOD={env!r}: this platform has {', '.join(methods)}")
|
|
629
|
+
return env
|
|
630
|
+
if sys.platform.startswith("linux") and "fork" in methods and threading.active_count() == 1:
|
|
631
|
+
return "fork"
|
|
632
|
+
return "spawn"
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
CHUNK = 16 # files a worker takes at a time; small, so that few files are in flight when a worker dies
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _timeout() -> float:
|
|
639
|
+
"""Seconds a worker may spend on one batch of files (or, retried alone, one file) before it is taken to be
|
|
640
|
+
stuck and stopped. LEYLINE_PARSE_TIMEOUT changes it."""
|
|
641
|
+
try:
|
|
642
|
+
return max(1.0, float(os.environ.get("LEYLINE_PARSE_TIMEOUT", "") or 600))
|
|
643
|
+
except ValueError:
|
|
644
|
+
return 600.0
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _parse_all(root: Path, repo: str, work: list, died: Optional[list] = None, keep_ext: frozenset = frozenset()):
|
|
648
|
+
"""Parse every file, across processes when there are enough files to pay for starting them.
|
|
649
|
+
Results come back in the order given, so the index is the same however many processes ran.
|
|
650
|
+
A worker that dies (a crash inside a parser, or the system killing it for memory) or gets stuck loses only the
|
|
651
|
+
file it was on: the files that were in flight are parsed again one at a time, each in a fresh process, and the
|
|
652
|
+
one that kills or stalls its process is reported as failed (and named in `died`)."""
|
|
653
|
+
jobs = _jobs()
|
|
654
|
+
items = [(str(root), repo, w, w[0] in keep_ext) for w in work]
|
|
655
|
+
if jobs == 1 or len(items) < PARALLEL_MIN_FILES:
|
|
656
|
+
yield from map(_parse_one, items)
|
|
657
|
+
return
|
|
658
|
+
import multiprocessing
|
|
659
|
+
from collections import deque
|
|
660
|
+
from concurrent.futures import ProcessPoolExecutor, TimeoutError as Stuck
|
|
661
|
+
from concurrent.futures.process import BrokenProcessPool
|
|
662
|
+
ctx = multiprocessing.get_context(_start_method())
|
|
663
|
+
limit = _timeout()
|
|
664
|
+
|
|
665
|
+
def pool(n):
|
|
666
|
+
return ProcessPoolExecutor(max_workers=n, mp_context=ctx, initializer=_worker_init)
|
|
667
|
+
|
|
668
|
+
chunks = [items[i:i + CHUNK] for i in range(0, len(items), CHUNK)]
|
|
669
|
+
pos = 0
|
|
670
|
+
worked = {"any": False} # whether a worker has ever returned a result in this run
|
|
671
|
+
while pos < len(chunks):
|
|
672
|
+
ahead: deque = deque()
|
|
673
|
+
nxt = pos
|
|
674
|
+
ex = pool(jobs)
|
|
675
|
+
try:
|
|
676
|
+
while pos < len(chunks):
|
|
677
|
+
# A few chunks per worker are handed out ahead of the one awaited: enough to keep every worker
|
|
678
|
+
# busy, few enough that when one dies the file that killed it is among them.
|
|
679
|
+
while nxt < len(chunks) and len(ahead) < 4 * jobs:
|
|
680
|
+
ahead.append(ex.submit(_parse_chunk, chunks[nxt]))
|
|
681
|
+
nxt += 1
|
|
682
|
+
try:
|
|
683
|
+
done = ahead[0].result(timeout=limit)
|
|
684
|
+
except Stuck:
|
|
685
|
+
_kill(ex)
|
|
686
|
+
raise BrokenProcessPool("stuck") from None
|
|
687
|
+
ahead.popleft()
|
|
688
|
+
pos += 1
|
|
689
|
+
worked["any"] = True
|
|
690
|
+
yield from done
|
|
691
|
+
except BrokenProcessPool:
|
|
692
|
+
suspects = [it for c in chunks[pos:nxt] for it in c]
|
|
693
|
+
ex.shutdown(wait=True)
|
|
694
|
+
yield from _parse_alone(suspects, pool, died, limit, worked)
|
|
695
|
+
if worked.get("broken"): # workers cannot run here at all: the rest is parsed in this process
|
|
696
|
+
yield from map(_parse_one, [it for c in chunks[nxt:] for it in c])
|
|
697
|
+
return
|
|
698
|
+
pos = nxt
|
|
699
|
+
finally:
|
|
700
|
+
ex.shutdown(wait=True)
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def _worker_init() -> None:
|
|
704
|
+
# Ctrl+C reaches every process in the group; the main process stops the run, and the workers stay quiet
|
|
705
|
+
# instead of each printing a traceback.
|
|
706
|
+
import signal
|
|
707
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _kill(ex) -> None:
|
|
711
|
+
for proc in list(getattr(ex, "_processes", {}).values()): # the pool has no public way to stop a busy worker
|
|
712
|
+
try:
|
|
713
|
+
proc.kill()
|
|
714
|
+
except Exception:
|
|
715
|
+
pass
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def _parse_chunk(items):
|
|
719
|
+
return [_parse_one(it) for it in items]
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def _parse_alone(items, pool, died, limit, worked):
|
|
723
|
+
"""Parse each file in a one-worker pool, starting a new pool after a file kills or stalls one."""
|
|
724
|
+
from concurrent.futures import TimeoutError as Stuck
|
|
725
|
+
from concurrent.futures.process import BrokenProcessPool
|
|
726
|
+
|
|
727
|
+
def failed(it, why):
|
|
728
|
+
f, ext, mod_dir, mod_id = it[2]
|
|
729
|
+
print(f"leyline: {why}: {f}; it is left out", file=sys.stderr)
|
|
730
|
+
if died is not None:
|
|
731
|
+
died.append(f)
|
|
732
|
+
try:
|
|
733
|
+
data = read_source(Path(it[0]) / f)
|
|
734
|
+
loc, sha = data.count(b"\n") + 1, hashlib.sha1(data).hexdigest()
|
|
735
|
+
except OSError:
|
|
736
|
+
loc, sha = 1, None
|
|
737
|
+
return f, ext, mod_dir, mod_id, loc, sha, None, why, b""
|
|
738
|
+
ex = pool(1)
|
|
739
|
+
# Files whose worker died before any worker in this run had finished a file: until one does, it is not known
|
|
740
|
+
# whether the files kill the workers or the workers cannot run here at all.
|
|
741
|
+
lost: list = []
|
|
742
|
+
try:
|
|
743
|
+
for k, it in enumerate(items):
|
|
744
|
+
if len(lost) >= 3:
|
|
745
|
+
print("leyline: worker processes are not working here; parsing in this process instead",
|
|
746
|
+
file=sys.stderr)
|
|
747
|
+
worked["broken"] = True
|
|
748
|
+
yield from map(_parse_one, [x for x, _ in lost] + list(items[k:]))
|
|
749
|
+
return
|
|
750
|
+
why = None
|
|
751
|
+
try:
|
|
752
|
+
got = ex.submit(_parse_one, it).result(timeout=limit)
|
|
753
|
+
except Stuck:
|
|
754
|
+
why = f"parsing took longer than {limit:g} s"
|
|
755
|
+
_kill(ex)
|
|
756
|
+
except BrokenProcessPool:
|
|
757
|
+
why = "the parser process died on this file"
|
|
758
|
+
if why is not None:
|
|
759
|
+
ex.shutdown(wait=True)
|
|
760
|
+
ex = pool(1)
|
|
761
|
+
if not worked["any"]:
|
|
762
|
+
lost.append((it, why))
|
|
763
|
+
continue
|
|
764
|
+
got = failed(it, why)
|
|
765
|
+
else:
|
|
766
|
+
worked["any"] = True
|
|
767
|
+
for x, w in lost: # a worker can run, so those files did kill theirs
|
|
768
|
+
yield failed(x, w)
|
|
769
|
+
lost = []
|
|
770
|
+
yield got
|
|
771
|
+
for x, w in lost:
|
|
772
|
+
yield failed(x, w)
|
|
773
|
+
finally:
|
|
774
|
+
ex.shutdown(wait=True)
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
class FlowSteps:
|
|
778
|
+
"""The steps of every flow, packed into arrays: as tuples they were the largest thing in memory on a large
|
|
779
|
+
repository (millions of them). Iterates as (flow id, seq, depth, callable id, via, site line, parent seq)."""
|
|
780
|
+
|
|
781
|
+
def __init__(self):
|
|
782
|
+
self.flows: list[tuple] = [] # (flow id, index of its first step)
|
|
783
|
+
self.callable: list[str] = [] # the node ids themselves, shared with the nodes
|
|
784
|
+
# depth is at most the walk's max_depth, a parent is a step of the same flow, and a line is under 10**9
|
|
785
|
+
self.depth, self.via, self.line, self.parent = array("h"), array("b"), array("i"), array("i")
|
|
786
|
+
self.via_names: list[str] = []
|
|
787
|
+
self._via: dict[str, int] = {}
|
|
788
|
+
|
|
789
|
+
def add(self, flow_id: str, steps: list[tuple]) -> None:
|
|
790
|
+
"""steps: (seq, depth, callable id, via, site line or None, parent seq or None), seq counting from 0."""
|
|
791
|
+
self.flows.append((flow_id, len(self.callable)))
|
|
792
|
+
for _seq, depth, c, via, line, parent in steps:
|
|
793
|
+
v = self._via.get(via)
|
|
794
|
+
if v is None:
|
|
795
|
+
v = self._via[via] = len(self.via_names)
|
|
796
|
+
self.via_names.append(via)
|
|
797
|
+
self.callable.append(c)
|
|
798
|
+
self.depth.append(depth)
|
|
799
|
+
self.via.append(v)
|
|
800
|
+
self.line.append(-1 if line is None else line)
|
|
801
|
+
self.parent.append(-1 if parent is None else parent)
|
|
802
|
+
|
|
803
|
+
def __len__(self) -> int:
|
|
804
|
+
return len(self.callable)
|
|
805
|
+
|
|
806
|
+
def ids(self):
|
|
807
|
+
"""Every flow id, then every callable id, in step order: the ids the rows name, as store._keys takes them."""
|
|
808
|
+
return itertools.chain((fid for fid, _ in self.flows), self.callable)
|
|
809
|
+
|
|
810
|
+
def __iter__(self):
|
|
811
|
+
ends = [start for _, start in self.flows[1:]] + [len(self.callable)]
|
|
812
|
+
for (fid, start), end in zip(self.flows, ends):
|
|
813
|
+
for i in range(start, end):
|
|
814
|
+
line, parent = self.line[i], self.parent[i]
|
|
815
|
+
yield (fid, i - start, self.depth[i], self.callable[i], self.via_names[self.via[i]],
|
|
816
|
+
None if line < 0 else line, None if parent < 0 else parent)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def flow_ids(starts: list) -> list:
|
|
820
|
+
"""(start, kind, detail) -> (start, kind, detail, flow id), each start walked once. A flow is `flow:<start>`;
|
|
821
|
+
a start that is also another kind of start (an entry point that is a test too) gets `flow:<start>#<kind>` for
|
|
822
|
+
the later ones, so neither flow replaces the other."""
|
|
823
|
+
out, taken = [], set()
|
|
824
|
+
for s in dict.fromkeys(starts):
|
|
825
|
+
start, kind, _ = s
|
|
826
|
+
fid, n = f"flow:{start}", 2
|
|
827
|
+
if fid in taken:
|
|
828
|
+
fid = f"flow:{start}#{kind}"
|
|
829
|
+
while fid in taken:
|
|
830
|
+
fid, n = f"flow:{start}#{kind}{n}", n + 1
|
|
831
|
+
taken.add(fid)
|
|
832
|
+
out.append((*s, fid))
|
|
833
|
+
return out
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _repo_of(node_id: str) -> str:
|
|
837
|
+
"""Every node id starts with its repo id: flask:python:..., flask:file:..., flask:module:..."""
|
|
838
|
+
return node_id.split(":", 1)[0]
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
_WILDCARD = re.compile(r"\*\w*|\(\.\*\)|:\w+(\*|\+|\(\.\*\))|\{\*\*?\w*\}")
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _wildcard(segment: str) -> bool:
|
|
845
|
+
"""A route segment that takes the rest of the path: Fastify's and Express's `*`, Express's `(.*)`, `:path*` and
|
|
846
|
+
`*name`, ASP.NET's `{*path}`."""
|
|
847
|
+
return bool(_WILDCARD.fullmatch(segment))
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _segment_fits(route: str, request: str) -> bool:
|
|
851
|
+
"""One segment of a route against one of a request: equal, or a parameter of the route (ASP.NET's [controller]
|
|
852
|
+
is the class's name, which clients write in lower case)."""
|
|
853
|
+
return route == request or route.startswith(("<", "{", ":")) or route.lower() == request.lower()
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def _fits_wildcard(route: list, request: list) -> bool:
|
|
857
|
+
"""Does a request's path fit a route that ends in a wildcard? The wildcard takes one segment or more; a hole in
|
|
858
|
+
the request (`/api/lore/${rel}`, written {}) may itself be several segments, so it can cover the route's
|
|
859
|
+
parameters up to and including the wildcard. A hole never stands in for a segment the route spells out."""
|
|
860
|
+
k = len(route) - 1
|
|
861
|
+
|
|
862
|
+
def go(i: int, j: int) -> bool:
|
|
863
|
+
if i == k:
|
|
864
|
+
return j < len(request)
|
|
865
|
+
if j == len(request) or not _segment_fits(route[i], request[j]):
|
|
866
|
+
return False
|
|
867
|
+
if go(i + 1, j + 1):
|
|
868
|
+
return True
|
|
869
|
+
if request[j] != "{}":
|
|
870
|
+
return False
|
|
871
|
+
i2 = i + 1
|
|
872
|
+
while i2 < k and route[i2].startswith(("<", "{", ":")):
|
|
873
|
+
if go(i2 + 1, j + 1):
|
|
874
|
+
return True
|
|
875
|
+
i2 += 1
|
|
876
|
+
return i2 == k and j == len(request) - 1
|
|
877
|
+
return go(0, 0)
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
class Indexer:
|
|
881
|
+
def __init__(self, root: str | Path, repo_id: Optional[str] = None,
|
|
882
|
+
others: Optional[list[tuple[str | Path, str]]] = None):
|
|
883
|
+
"""`others` are more (root, repo id) pairs indexed in the same run, a workspace: a name in one
|
|
884
|
+
repository resolves to its declaration in another. Node ids keep their own repo's prefix."""
|
|
885
|
+
self.root = Path(root).resolve()
|
|
886
|
+
self.repo = repo_id or self.root.name
|
|
887
|
+
self.repos: dict[str, Path] = {self.repo: self.root}
|
|
888
|
+
for r, rid in others or ():
|
|
889
|
+
if rid in self.repos or ":" in rid:
|
|
890
|
+
raise ValueError(f"repo id {rid!r} is used twice or contains ':'")
|
|
891
|
+
self.repos[rid] = Path(r).resolve()
|
|
892
|
+
self.commit = _git(self.root, "rev-parse", "HEAD")
|
|
893
|
+
self.commits = {rid: _git(r, "rev-parse", "HEAD") for rid, r in self.repos.items()}
|
|
894
|
+
self.nodes: dict[str, Node] = {}
|
|
895
|
+
self.edges: list[Edge] = []
|
|
896
|
+
self.calls: list[tuple] = []
|
|
897
|
+
self.results: dict[str, FileResult] = {} # file id -> adapter output
|
|
898
|
+
# On a large repository, file id -> its calls and field uses, packed (leyline.packing); see _open
|
|
899
|
+
self._packed: dict[str, bytes] = {}
|
|
900
|
+
self._packed_decls: dict[str, bytes] = {}
|
|
901
|
+
self._opened: Optional[list] = None # [file id, result, changed] of the one file unpacked now
|
|
902
|
+
self._ckeys: Optional[dict] = None
|
|
903
|
+
self._file_index: dict[str, int] = {}
|
|
904
|
+
self.file_lang: dict[str, str] = {}
|
|
905
|
+
self.file_of_path: dict[str, str] = {}
|
|
906
|
+
self.keep_ext: set[str] = set() # file ids whose node ids keep the file's extension (_ext_clashes)
|
|
907
|
+
self._decl_cache: dict[str, dict] = {}
|
|
908
|
+
self._read_decls: dict[str, dict] = {} # file id -> node id -> (span, declarations read in the parse)
|
|
909
|
+
self._text_cache: dict[str, list] = {}
|
|
910
|
+
self.stats: dict[str, Counter] = defaultdict(Counter)
|
|
911
|
+
self.flows: list[tuple] = []
|
|
912
|
+
self.call_col: dict[tuple, int] = {}
|
|
913
|
+
self.flow_steps = FlowSteps()
|
|
914
|
+
self.channel_stats: dict[str, Counter] = defaultdict(Counter)
|
|
915
|
+
self.project_refs: dict[str, set] = defaultdict(set)
|
|
916
|
+
self._vis_cache: dict[str, set] = {}
|
|
917
|
+
self._sets: dict[tuple, set] = {} # see _file_set
|
|
918
|
+
self._name_ix: dict[tuple, dict] = {}
|
|
919
|
+
self._loose_reach: dict[str, set] = {}
|
|
920
|
+
self.exact_mode = "off" # off | auto | roslyn | scip
|
|
921
|
+
self.scip_paths: list[str] = []
|
|
922
|
+
self.listed: dict = {} # repo -> its Listing, or its files, when the caller has listed them (see scan)
|
|
923
|
+
self.exact_stats: dict[str, dict] = {}
|
|
924
|
+
self.keep_results = True # False: run() lets go of the adapters' output and the resolvers' caches before writing
|
|
925
|
+
self.parse_cache = None # leyline.incremental: parse output kept from the last run, by content hash
|
|
926
|
+
|
|
927
|
+
def _apply_exact(self) -> None:
|
|
928
|
+
"""Let a compiler overrule the syntax resolvers where one is available."""
|
|
929
|
+
if self.exact_mode == "off":
|
|
930
|
+
return
|
|
931
|
+
from . import exact
|
|
932
|
+
multi = len(self.repos) > 1
|
|
933
|
+
if self.exact_mode in ("auto", "roslyn") and any(v == "csharp" for v in self.file_lang.values()):
|
|
934
|
+
if multi:
|
|
935
|
+
records, info = [], {"status": "skipped", "reason": "not yet run for a workspace of several repositories"}
|
|
936
|
+
else:
|
|
937
|
+
try:
|
|
938
|
+
records, info = exact.roslyn(self)
|
|
939
|
+
except RuntimeError as exc:
|
|
940
|
+
records, info = [], {"status": "failed", "reason": str(exc)[-600:]}
|
|
941
|
+
if records:
|
|
942
|
+
info.update(exact.apply(self, records, "roslyn"))
|
|
943
|
+
self.exact_stats["exact:roslyn"] = info
|
|
944
|
+
paths = [(p, None) for p in self.scip_paths]
|
|
945
|
+
if self.exact_mode in ("auto", "scip") and not paths:
|
|
946
|
+
paths = [(str(p), repo) for repo, root in self.repos.items()
|
|
947
|
+
for p in (root / "index.scip", root / ".leyline" / "index.scip") if p.is_file()]
|
|
948
|
+
by_repo = {}
|
|
949
|
+
for path, repo in paths:
|
|
950
|
+
try:
|
|
951
|
+
if repo is None: # a SCIP index names the directory it was made in; that says whose it is
|
|
952
|
+
repo = next((r for r, root in self.repos.items() if exact.scip_root(path) == root), self.repo) if multi else self.repo
|
|
953
|
+
info = {"status": "ok", "file": path, **exact.apply(self, exact.scip(path, self.repos[repo]), "scip",
|
|
954
|
+
repo if multi else None)}
|
|
955
|
+
except Exception as exc:
|
|
956
|
+
info = {"status": "failed", "reason": str(exc)[-600:]}
|
|
957
|
+
self.exact_stats["exact:scip"] = info
|
|
958
|
+
by_repo[repo] = info
|
|
959
|
+
if multi and by_repo:
|
|
960
|
+
ok = any(i["status"] == "ok" for i in by_repo.values())
|
|
961
|
+
self.exact_stats["exact:scip"] = {"status": "ok" if ok else "failed", "by_repo": by_repo}
|
|
962
|
+
|
|
963
|
+
# -- public --------------------------------------------------------------
|
|
964
|
+
def _timed(self, name: str, fn, *args) -> None:
|
|
965
|
+
start = time.perf_counter()
|
|
966
|
+
fn(*args)
|
|
967
|
+
self.timing[name] = round(self.timing.get(name, 0) + time.perf_counter() - start, 3)
|
|
968
|
+
|
|
969
|
+
def run(self, con) -> dict:
|
|
970
|
+
self.timing: dict[str, float] = {}
|
|
971
|
+
started = time.perf_counter()
|
|
972
|
+
self.files_of, self.skipped, self.failed = {}, {}, {}
|
|
973
|
+
for rid, root in self.repos.items():
|
|
974
|
+
got = self.listed.get(rid) # listed already by the caller: its files, or the whole listing
|
|
975
|
+
listing = got if isinstance(got, Listing) else scan(root, got)
|
|
976
|
+
self.files_of[rid], self.skipped[rid] = listing.files, listing.skipped
|
|
977
|
+
if listing.why_walk:
|
|
978
|
+
print(f"leyline: git would not list the files of {root} ({listing.why_walk}); reading the directory"
|
|
979
|
+
" instead, with the common .gitignore patterns applied", file=sys.stderr)
|
|
980
|
+
elif listing.how == "walk":
|
|
981
|
+
print(f"leyline: {root} is not a git repository: reading every file under it, with the common"
|
|
982
|
+
" .gitignore patterns applied", file=sys.stderr)
|
|
983
|
+
_report_skipped(rid, listing.skipped)
|
|
984
|
+
for repo, root in self.repos.items():
|
|
985
|
+
self._parse_repo(repo, root, self.files_of[repo])
|
|
986
|
+
self.timing["parse"] = round(time.perf_counter() - started, 3)
|
|
987
|
+
|
|
988
|
+
def resolve():
|
|
989
|
+
for repo, files in self.files_of.items():
|
|
990
|
+
self._projects(files, repo)
|
|
991
|
+
self._attach_methods()
|
|
992
|
+
self._build_indexes()
|
|
993
|
+
self._resolve_imports()
|
|
994
|
+
self._script_entries()
|
|
995
|
+
self._resolve_types()
|
|
996
|
+
self._resolve_overrides()
|
|
997
|
+
self._resolve_calls()
|
|
998
|
+
self._resolve_events()
|
|
999
|
+
self._resolve_fields()
|
|
1000
|
+
self._timed("resolve", resolve)
|
|
1001
|
+
self._timed("compiler", self._apply_exact)
|
|
1002
|
+
|
|
1003
|
+
def channels():
|
|
1004
|
+
self._resolve_spawns()
|
|
1005
|
+
self._resolve_endpoints()
|
|
1006
|
+
from . import channels as more
|
|
1007
|
+
more.resolve(self) # dependency injection, queues, databases, RPC
|
|
1008
|
+
self._timed("channels", channels)
|
|
1009
|
+
if not self.keep_results:
|
|
1010
|
+
# Flows and the write are left, which read nodes, edges, calls (with their columns) and flows. What the
|
|
1011
|
+
# parse and the resolvers held (GBs on a large repository) is let go first, so the flows' lists and the
|
|
1012
|
+
# write's reuse that memory instead of adding to it.
|
|
1013
|
+
for held in (self.results, self._read_decls, self._decl_cache, self._name_ix, self._sets, self._text_cache,
|
|
1014
|
+
getattr(self, "_chain_memo", {}), self._packed, self._packed_decls,
|
|
1015
|
+
getattr(self, "_decls_unpacked", {})):
|
|
1016
|
+
held.clear()
|
|
1017
|
+
self._timed("flows", self._build_flows)
|
|
1018
|
+
if not self.keep_results:
|
|
1019
|
+
for held in (self._vis_cache, self.call_col, self._loose_reach):
|
|
1020
|
+
held.clear()
|
|
1021
|
+
self._timed("write", self._write, con)
|
|
1022
|
+
return {k: dict(v) for k, v in self.stats.items()}
|
|
1023
|
+
|
|
1024
|
+
def _parse_repo(self, repo: str, root: Path, files: list[str]) -> None:
|
|
1025
|
+
module_dirs = _module_dirs(files)
|
|
1026
|
+
self._add(Node(id=repo, kind="repo", name=repo, path="",
|
|
1027
|
+
attrs={"url": _git(root, "remote", "get-url", "origin"),
|
|
1028
|
+
"branch": _git(root, "rev-parse", "--abbrev-ref", "HEAD")}))
|
|
1029
|
+
work = []
|
|
1030
|
+
for f in files:
|
|
1031
|
+
ext = "." + f.rsplit(".", 1)[-1] if "." in f else ""
|
|
1032
|
+
adapter = BY_EXTENSION.get(ext)
|
|
1033
|
+
if adapter is None:
|
|
1034
|
+
continue
|
|
1035
|
+
mod_dir = _module_for(f, module_dirs)
|
|
1036
|
+
mod_id = self._module(mod_dir, files, repo)
|
|
1037
|
+
work.append((f, ext, mod_dir, mod_id))
|
|
1038
|
+
# With a parse cache (leyline.incremental), a file whose content and module are as they were last time is
|
|
1039
|
+
# not parsed again. Either way the files are taken in the listed order, so the index is the same.
|
|
1040
|
+
cache = self.parse_cache
|
|
1041
|
+
keep_ext = _ext_clashes(work)
|
|
1042
|
+
self.keep_ext.update(f"{repo}:file:{f}" for f in keep_ext)
|
|
1043
|
+
cached = cache.lookup(repo, root, work, keep_ext) if cache is not None else {}
|
|
1044
|
+
fresh = _parse_all(root, repo, [w for w in work if w[0] not in cached], keep_ext=keep_ext)
|
|
1045
|
+
failed_here = self.failed.setdefault(repo, [])
|
|
1046
|
+
packs = packing.packs(len(work))
|
|
1047
|
+
for w in work:
|
|
1048
|
+
got = cached.pop(w[0], None) # let go as it is used: on a large repository the blobs add up
|
|
1049
|
+
fresh_one = got is None
|
|
1050
|
+
if fresh_one:
|
|
1051
|
+
got = next(fresh)
|
|
1052
|
+
f, ext, mod_dir, mod_id, loc, sha, blob, failed, toks = got
|
|
1053
|
+
if cache is not None:
|
|
1054
|
+
cache.keep(f"{repo}:file:{f}", got, fresh_one, f in keep_ext)
|
|
1055
|
+
del got
|
|
1056
|
+
res, heavy, decls = packing.unpack(blob) if blob is not None else (None, None, None)
|
|
1057
|
+
blob = None
|
|
1058
|
+
adapter = BY_EXTENSION[ext]
|
|
1059
|
+
file_id = f"{repo}:file:{f}"
|
|
1060
|
+
self._add(Node(id=file_id, kind="file", name=f.rsplit("/", 1)[-1], parent_id=mod_id,
|
|
1061
|
+
language=adapter.LANGUAGE, path=f, span_start=1, span_end=loc,
|
|
1062
|
+
content_hash=sha, attrs={"loc": loc}))
|
|
1063
|
+
if res is None: # one bad file must not sink the run
|
|
1064
|
+
self.stats[adapter.NAME]["files_failed"] += 1
|
|
1065
|
+
failed_here.append((f, failed))
|
|
1066
|
+
print(f"leyline: failed to parse {f}: {failed}", file=sys.stderr)
|
|
1067
|
+
continue
|
|
1068
|
+
self.results[file_id] = res
|
|
1069
|
+
# Python's calls are kept unpacked: the argument-flow rule holds on to them after their file's turn.
|
|
1070
|
+
if packs and adapter.LANGUAGE != "python":
|
|
1071
|
+
self._packed[file_id] = heavy
|
|
1072
|
+
if decls is not None:
|
|
1073
|
+
self._packed_decls[file_id] = decls
|
|
1074
|
+
else:
|
|
1075
|
+
res.calls, res.field_uses = packing.unpack_heavy(heavy)
|
|
1076
|
+
if decls is not None:
|
|
1077
|
+
self._read_decls[file_id] = packing.unpack_decls(decls)
|
|
1078
|
+
self.file_of_path[f] = file_id
|
|
1079
|
+
self.file_lang[file_id] = adapter.LANGUAGE
|
|
1080
|
+
self.stats[adapter.NAME]["files"] += 1
|
|
1081
|
+
for n in res.nodes:
|
|
1082
|
+
self._add(n)
|
|
1083
|
+
if res.calls is not None:
|
|
1084
|
+
self._share_ids(res)
|
|
1085
|
+
self.edges.extend(res.edges)
|
|
1086
|
+
next(fresh, None) # lets the worker pool, if one was started, shut down
|
|
1087
|
+
|
|
1088
|
+
# -- packed calls (leyline.packing) -----------------------------------------
|
|
1089
|
+
def _open(self, fid: str, res: FileResult) -> FileResult:
|
|
1090
|
+
"""The file's result with its calls and field uses in place. Packed ones are unpacked here, one file at a
|
|
1091
|
+
time: opening a file lets go of the one opened before (packing it again if it was changed)."""
|
|
1092
|
+
if res.calls is not None:
|
|
1093
|
+
return res
|
|
1094
|
+
self._shut()
|
|
1095
|
+
res.calls, res.field_uses = packing.unpack_heavy(self._packed[fid])
|
|
1096
|
+
self._share_ids(res)
|
|
1097
|
+
self._opened = [fid, res, False]
|
|
1098
|
+
return res
|
|
1099
|
+
|
|
1100
|
+
def _share_ids(self, res: FileResult) -> None:
|
|
1101
|
+
"""Point each call's and field use's caller at its node's own id string. Unpacked apart from the nodes,
|
|
1102
|
+
each was a copy of it, and every call row made from the call kept that copy."""
|
|
1103
|
+
nodes = self.nodes
|
|
1104
|
+
for group in (res.calls, res.field_uses):
|
|
1105
|
+
for c in group:
|
|
1106
|
+
n = nodes.get(c.src_id)
|
|
1107
|
+
if n is not None:
|
|
1108
|
+
c.src_id = n.id
|
|
1109
|
+
|
|
1110
|
+
def _shut(self) -> None:
|
|
1111
|
+
if self._opened is None:
|
|
1112
|
+
return
|
|
1113
|
+
fid, res, changed = self._opened
|
|
1114
|
+
if changed:
|
|
1115
|
+
self._packed[fid] = packing.pack_heavy(res.calls, res.field_uses)
|
|
1116
|
+
res.calls = res.field_uses = None
|
|
1117
|
+
self._opened = self._ckeys = None
|
|
1118
|
+
|
|
1119
|
+
def _ckey(self, call: CallSite):
|
|
1120
|
+
"""What _chain_memo keeps a call's results under: the call itself while it lives, which for a packed file
|
|
1121
|
+
is only while the file is open, so there a key that names the same call each time it is unpacked."""
|
|
1122
|
+
if self._opened is None:
|
|
1123
|
+
return id(call)
|
|
1124
|
+
if self._ckeys is None:
|
|
1125
|
+
fid = self._opened[0]
|
|
1126
|
+
self._ckeys = packing.call_keys(self._opened[1], self._file_index.setdefault(fid, len(self._file_index)))
|
|
1127
|
+
return self._ckeys.get(id(call), id(call))
|
|
1128
|
+
|
|
1129
|
+
def _decls_read(self, fid: Optional[str]) -> dict:
|
|
1130
|
+
"""node id -> (span, declarations) the parse read in a file, unpacked for the last file asked about."""
|
|
1131
|
+
got = self._read_decls.get(fid)
|
|
1132
|
+
if got is None and fid in self._packed_decls:
|
|
1133
|
+
cache = self.__dict__.setdefault("_decls_unpacked", {})
|
|
1134
|
+
got = cache.get(fid)
|
|
1135
|
+
if got is None:
|
|
1136
|
+
if len(cache) >= 2:
|
|
1137
|
+
cache.clear()
|
|
1138
|
+
got = cache[fid] = packing.unpack_decls(self._packed_decls[fid])
|
|
1139
|
+
return got or {}
|
|
1140
|
+
|
|
1141
|
+
# -- structure -----------------------------------------------------------
|
|
1142
|
+
def _add(self, n: Node) -> None:
|
|
1143
|
+
if n.id in self.nodes:
|
|
1144
|
+
# Partial classes: keep the first declaration, note the extra file.
|
|
1145
|
+
first = self.nodes[n.id]
|
|
1146
|
+
if n.kind == "type" and n.path != first.path:
|
|
1147
|
+
first.attrs.setdefault("also_in", []).append(n.path)
|
|
1148
|
+
elif n.path == first.path and n.span_start and first.span_start and n.kind not in ("file", "module", "repo"):
|
|
1149
|
+
# One name defined twice in a file (if/else or try/except, a property's getter and setter): which one
|
|
1150
|
+
# runs is not known from the text, so it is one node over both, listing each, its text hash reading
|
|
1151
|
+
# every body.
|
|
1152
|
+
spans = first.attrs.setdefault("definitions", [[first.span_start, first.span_end or first.span_start]])
|
|
1153
|
+
spans.append([n.span_start, n.span_end or n.span_start])
|
|
1154
|
+
first.span_start = min(first.span_start, n.span_start)
|
|
1155
|
+
first.span_end = max(first.span_end or first.span_start, n.span_end or n.span_start)
|
|
1156
|
+
if first.content_hash and n.content_hash:
|
|
1157
|
+
first.content_hash = hashlib.sha1(f"{first.content_hash}|{n.content_hash}".encode()).hexdigest()[:16]
|
|
1158
|
+
return
|
|
1159
|
+
self.nodes[n.id] = n
|
|
1160
|
+
|
|
1161
|
+
def _module(self, mod_dir: str, files: list[str], repo: Optional[str] = None) -> str:
|
|
1162
|
+
repo = repo or self.repo
|
|
1163
|
+
mid = f"{repo}:module:{mod_dir or '.'}"
|
|
1164
|
+
if mid not in self.nodes:
|
|
1165
|
+
marker = self._markers(files).get(mod_dir)
|
|
1166
|
+
self._add(Node(id=mid, kind="module", name=mod_dir.rsplit("/", 1)[-1] or repo,
|
|
1167
|
+
parent_id=repo, path=mod_dir, attrs={"marker": marker}))
|
|
1168
|
+
return mid
|
|
1169
|
+
|
|
1170
|
+
def _markers(self, files: list[str]) -> dict:
|
|
1171
|
+
"""directory -> the first project marker in it, for a list of files. Made once per list: looking through
|
|
1172
|
+
every file for each new module was quadratic (minutes on a repository of tens of thousands of files)."""
|
|
1173
|
+
got = getattr(self, "_marker_memo", None)
|
|
1174
|
+
if got is None or got[0] is not files:
|
|
1175
|
+
out: dict[str, str] = {}
|
|
1176
|
+
for f in files:
|
|
1177
|
+
d, _, base = f.rpartition("/")
|
|
1178
|
+
if d not in out and (f.endswith(".csproj") or base in MODULE_MARKERS):
|
|
1179
|
+
out[d] = base
|
|
1180
|
+
got = self._marker_memo = (files, out)
|
|
1181
|
+
return got[1]
|
|
1182
|
+
|
|
1183
|
+
def _projects(self, files: list[str], repo: Optional[str] = None) -> None:
|
|
1184
|
+
"""Project files give exact module-to-module and module-to-package edges."""
|
|
1185
|
+
repo = repo or self.repo
|
|
1186
|
+
root = self.repos[repo]
|
|
1187
|
+
for f in files:
|
|
1188
|
+
if not f.endswith(".csproj"):
|
|
1189
|
+
continue
|
|
1190
|
+
d = f.rpartition("/")[0]
|
|
1191
|
+
mid = f"{repo}:module:{d or '.'}"
|
|
1192
|
+
if mid not in self.nodes:
|
|
1193
|
+
continue
|
|
1194
|
+
try:
|
|
1195
|
+
text = (root / f).read_text(encoding="utf-8", errors="replace")
|
|
1196
|
+
except OSError:
|
|
1197
|
+
continue
|
|
1198
|
+
for ref in re.findall(r'<ProjectReference\s+Include="([^"]+)"', text):
|
|
1199
|
+
target = (Path(d) / ref.replace("\\", "/")).parent
|
|
1200
|
+
tdir = Path(*_normalize(target.parts)).as_posix() if target.parts else ""
|
|
1201
|
+
tid = f"{repo}:module:{tdir or '.'}"
|
|
1202
|
+
if target.parts[:1] == ("..",) or tid not in self.nodes:
|
|
1203
|
+
tid = self._module_elsewhere(root / target) or tid # a project in another repo of the workspace
|
|
1204
|
+
if tid in self.nodes:
|
|
1205
|
+
self.edges.append(Edge("imports", mid, tid, "exact", {"via": "ProjectReference"}))
|
|
1206
|
+
self.project_refs[mid].add(tid)
|
|
1207
|
+
for name, ver in re.findall(r'<PackageReference\s+Include="([^"]+)"(?:\s+Version="([^"]+)")?', text):
|
|
1208
|
+
xid = self._external("nuget", name, {"category": "package", "version": ver or None}, repo)
|
|
1209
|
+
self.edges.append(Edge("depends_on", mid, xid, "exact", {"version_range": ver or None}))
|
|
1210
|
+
sdk = re.search(r'<Project\s+Sdk="([^"/]+)(?:/([^"]+))?"', text)
|
|
1211
|
+
if sdk and sdk.group(1) != "Microsoft.NET.Sdk":
|
|
1212
|
+
xid = self._external("nuget", sdk.group(1), {"category": "sdk", "version": sdk.group(2)}, repo)
|
|
1213
|
+
self.edges.append(Edge("depends_on", mid, xid, "exact", {"version_range": sdk.group(2)}))
|
|
1214
|
+
tf = re.search(r"<TargetFramework>([^<]+)<", text)
|
|
1215
|
+
if tf:
|
|
1216
|
+
self.nodes[mid].attrs["target_framework"] = tf.group(1)
|
|
1217
|
+
|
|
1218
|
+
def _module_elsewhere(self, path: Path) -> Optional[str]:
|
|
1219
|
+
"""The module of another workspace repo that an absolute directory is, if any."""
|
|
1220
|
+
if len(self.repos) < 2:
|
|
1221
|
+
return None
|
|
1222
|
+
path = Path(os.path.normpath(path))
|
|
1223
|
+
for repo, root in self.repos.items():
|
|
1224
|
+
if path == root or root in path.parents:
|
|
1225
|
+
mid = f"{repo}:module:{path.relative_to(root).as_posix()}"
|
|
1226
|
+
return mid if mid in self.nodes else None
|
|
1227
|
+
return None
|
|
1228
|
+
|
|
1229
|
+
def _external(self, eco: str, name: str, attrs: Optional[dict] = None, repo: Optional[str] = None) -> str:
|
|
1230
|
+
repo = repo or self.repo
|
|
1231
|
+
xid = f"{repo}:ext:{eco}:{name}"
|
|
1232
|
+
if xid not in self.nodes:
|
|
1233
|
+
self._add(Node(id=xid, kind="external", name=name, parent_id=repo,
|
|
1234
|
+
attrs={"ecosystem": eco, **(attrs or {})}))
|
|
1235
|
+
return xid
|
|
1236
|
+
|
|
1237
|
+
# -- indexes -------------------------------------------------------------
|
|
1238
|
+
def _build_indexes(self) -> None:
|
|
1239
|
+
self.types_by_name: dict[tuple, list[str]] = defaultdict(list) # (lang, name) -> type ids
|
|
1240
|
+
self.members: dict[str, dict[str, list[Node]]] = defaultdict(lambda: defaultdict(list))
|
|
1241
|
+
self.by_name: dict[tuple, list[Node]] = defaultdict(list) # (lang, name) -> callables
|
|
1242
|
+
self.field_type: dict[str, dict[str, str]] = defaultdict(dict) # type id -> field -> type name
|
|
1243
|
+
self.field_types_global: dict[tuple, set] = defaultdict(set) # (lang, field) -> type names
|
|
1244
|
+
self.bases: dict[str, list[str]] = defaultdict(list)
|
|
1245
|
+
self.field_names: dict[str, set] = defaultdict(set)
|
|
1246
|
+
self.ns_modules: dict[str, set] = defaultdict(set) # C# namespace -> module ids
|
|
1247
|
+
self.py_modules: dict[str, dict] = defaultdict(dict) # repo -> python module path -> file id
|
|
1248
|
+
self.path_modules: dict[str, dict] = defaultdict(dict) # repo -> path without extension -> file id (TypeScript)
|
|
1249
|
+
self.py_importable: dict[str, str] = {} # python import name -> file id, across the workspace
|
|
1250
|
+
self.star_exports: dict[str, list] = defaultdict(list) # file -> files it re-exports everything from
|
|
1251
|
+
self.default_export: dict[str, str] = {} # file -> the node it exports as default
|
|
1252
|
+
self.packages: dict[str, dict] = {} # npm package name -> {"dir", "exports", "main"}
|
|
1253
|
+
for n in self.nodes.values():
|
|
1254
|
+
if n.attrs.get("is_default_export") and n.kind in ("callable", "type"):
|
|
1255
|
+
self.default_export[n.parent_id] = n.id
|
|
1256
|
+
self.file_of: dict[str, str] = {}
|
|
1257
|
+
self.field_calls: dict[tuple, str] = {} # (type id, field) -> the method whose result it holds
|
|
1258
|
+
self.py_vars: dict[str, dict] = {fid: res.var_types for fid, res in self.results.items() if res.var_types}
|
|
1259
|
+
for n in self.nodes.values():
|
|
1260
|
+
if n.kind == "type":
|
|
1261
|
+
self.types_by_name[(n.language, n.name)].append(n.id)
|
|
1262
|
+
elif n.kind == "callable":
|
|
1263
|
+
self.members[n.parent_id][n.name].append(n)
|
|
1264
|
+
self.by_name[(n.language, n.name)].append(n)
|
|
1265
|
+
elif n.kind == "field":
|
|
1266
|
+
self.field_names[n.parent_id].add(n.name)
|
|
1267
|
+
if n.attrs.get("type_call"):
|
|
1268
|
+
self.field_calls[(n.parent_id, n.name)] = n.attrs["type_call"]
|
|
1269
|
+
tn = n.attrs.get("type_name")
|
|
1270
|
+
if tn:
|
|
1271
|
+
self.field_type[n.parent_id][n.name] = tn
|
|
1272
|
+
self.field_types_global[(n.language, n.name)].add(tn)
|
|
1273
|
+
for fid, res in self.results.items():
|
|
1274
|
+
lang = self.file_lang[fid]
|
|
1275
|
+
mod_id = self.nodes[fid].parent_id
|
|
1276
|
+
for d in res.declares:
|
|
1277
|
+
if lang == "csharp":
|
|
1278
|
+
self.ns_modules[d].add(mod_id)
|
|
1279
|
+
elif lang == "python":
|
|
1280
|
+
self.py_modules[_repo_of(fid)][d] = fid
|
|
1281
|
+
else:
|
|
1282
|
+
self.path_modules[_repo_of(fid)][d] = fid
|
|
1283
|
+
for n in res.nodes:
|
|
1284
|
+
self.file_of[n.id] = fid
|
|
1285
|
+
# A package under a source root (src/flask) is imported by its own name (flask), not by its path.
|
|
1286
|
+
for repo in self.repos:
|
|
1287
|
+
py_files = {self.nodes[f].path: f for f in self.results if self.file_lang[f] == "python" and _repo_of(f) == repo}
|
|
1288
|
+
dirs_with_init = {p.rsplit("/", 1)[0] if "/" in p else "" for p in py_files if p.endswith("__init__.py")}
|
|
1289
|
+
for path, fid in sorted(py_files.items()):
|
|
1290
|
+
parts = path.split("/")
|
|
1291
|
+
start = len(parts) - 1
|
|
1292
|
+
while start > 0 and "/".join(parts[:start]) in dirs_with_init:
|
|
1293
|
+
start -= 1
|
|
1294
|
+
if start == len(parts) - 1 and "src" in parts[:-1]:
|
|
1295
|
+
# a namespace package (PEP 420, no __init__.py) in a src layout: src/corp/tools/x.py is corp.tools.x
|
|
1296
|
+
start = len(parts) - 1 - parts[:-1][::-1].index("src")
|
|
1297
|
+
if 0 < start < len(parts) - 1:
|
|
1298
|
+
self.py_modules[repo].setdefault(module_path("/".join(parts[start:])), fid)
|
|
1299
|
+
if len(self.repos) > 1 and start < len(parts) - 1:
|
|
1300
|
+
# Inside a package: another repository of the workspace can import it by this name.
|
|
1301
|
+
self.py_importable.setdefault(module_path("/".join(parts[start:])), fid)
|
|
1302
|
+
|
|
1303
|
+
# -- imports -------------------------------------------------------------
|
|
1304
|
+
def _resolve_imports(self) -> None:
|
|
1305
|
+
self.cs_usings: dict[str, set] = defaultdict(set) # file -> namespaces
|
|
1306
|
+
self.cs_static: dict[str, list] = defaultdict(list) # file -> type names from `using static`
|
|
1307
|
+
self.cs_alias: dict[str, dict] = defaultdict(dict) # file -> alias -> type name
|
|
1308
|
+
self.py_names: dict[str, dict] = defaultdict(dict) # file -> local name -> (file id, symbol|None)
|
|
1309
|
+
self.outside_imports: dict[str, set] = defaultdict(set) # file -> local names imported from outside the workspace
|
|
1310
|
+
self.import_targets: dict[str, set] = defaultdict(set) # file -> files it imports, by any form of import
|
|
1311
|
+
for repo, files in self.files_of.items():
|
|
1312
|
+
for f in files:
|
|
1313
|
+
if f.rsplit("/", 1)[-1] == "package.json":
|
|
1314
|
+
try:
|
|
1315
|
+
data = json.loads((self.repos[repo] / f).read_text(encoding="utf-8"))
|
|
1316
|
+
except (OSError, ValueError, RecursionError): # RecursionError: nested past the limit
|
|
1317
|
+
continue
|
|
1318
|
+
if isinstance(data, dict) and data.get("name"):
|
|
1319
|
+
self.packages[data["name"]] = {"dir": f.rpartition("/")[0], "exports": data.get("exports"), "repo": repo,
|
|
1320
|
+
"main": data.get("source") or data.get("module") or data.get("main") or data.get("types")}
|
|
1321
|
+
self._go_modules()
|
|
1322
|
+
seen = set()
|
|
1323
|
+
stdlib = getattr(sys, "stdlib_module_names", frozenset())
|
|
1324
|
+
self.py_local: dict[tuple, tuple] = {} # (function, name) -> what an import inside that function binds
|
|
1325
|
+
|
|
1326
|
+
def bind(fid: str, src: str, name: str, value: tuple) -> None:
|
|
1327
|
+
if src == fid:
|
|
1328
|
+
self.py_names[fid][name] = value
|
|
1329
|
+
return
|
|
1330
|
+
# An import inside a function binds the name there only: elsewhere in the file the same name may be a
|
|
1331
|
+
# local variable (`query = Query()` beside another function's `from .sql import query`).
|
|
1332
|
+
self.py_local[(src, name)] = value
|
|
1333
|
+
if value[0]:
|
|
1334
|
+
self.import_targets[fid].add(value[0])
|
|
1335
|
+
for fid, res in self.results.items():
|
|
1336
|
+
lang = self.file_lang[fid]
|
|
1337
|
+
for imp in res.imports:
|
|
1338
|
+
if lang == "csharp":
|
|
1339
|
+
if imp.alias:
|
|
1340
|
+
self.cs_alias[fid][imp.alias] = imp.target.rsplit(".", 1)[-1]
|
|
1341
|
+
if imp.is_static:
|
|
1342
|
+
self.cs_static[fid].append(imp.target.rsplit(".", 1)[-1])
|
|
1343
|
+
ns = imp.target
|
|
1344
|
+
mods = self.ns_modules.get(ns)
|
|
1345
|
+
if not mods and (imp.is_static or imp.alias):
|
|
1346
|
+
ns = imp.target.rsplit(".", 1)[0]
|
|
1347
|
+
mods = self.ns_modules.get(ns)
|
|
1348
|
+
self.cs_usings[fid].add(ns)
|
|
1349
|
+
vis = self._visible(fid)
|
|
1350
|
+
declared = bool(mods)
|
|
1351
|
+
if mods and vis is not None:
|
|
1352
|
+
# A namespace can be declared by several projects; only the referenced ones count.
|
|
1353
|
+
mods = {m for m in mods if m in vis or m == self.nodes[fid].parent_id}
|
|
1354
|
+
if mods:
|
|
1355
|
+
for m in sorted(mods):
|
|
1356
|
+
key = (fid, m, ns)
|
|
1357
|
+
if key not in seen and m != self.nodes[fid].parent_id:
|
|
1358
|
+
seen.add(key)
|
|
1359
|
+
self.edges.append(Edge("imports", fid, m, "exact", {"namespace": ns}))
|
|
1360
|
+
elif not declared:
|
|
1361
|
+
xid = self._external("dotnet", imp.target if not (imp.is_static or imp.alias) else ns,
|
|
1362
|
+
{"category": "namespace"}, _repo_of(fid))
|
|
1363
|
+
if (fid, xid) not in seen:
|
|
1364
|
+
seen.add((fid, xid))
|
|
1365
|
+
self.edges.append(Edge("imports", fid, xid, "exact"))
|
|
1366
|
+
else:
|
|
1367
|
+
target = self._find_module(fid, imp.target)
|
|
1368
|
+
on_path = False
|
|
1369
|
+
if not target and lang == "python" and imp.target.split(".")[0] not in stdlib:
|
|
1370
|
+
target = self._py_on_path(fid, imp.target)
|
|
1371
|
+
on_path = bool(target)
|
|
1372
|
+
subs = {} if target or lang != "python" else \
|
|
1373
|
+
{s: self._py_module(fid, f"{imp.target}.{s.partition(' as ')[0]}") for s in imp.symbols or ()}
|
|
1374
|
+
if any(subs.values()):
|
|
1375
|
+
# `from corp.tools import strings` where corp/tools is a namespace package (no __init__.py):
|
|
1376
|
+
# the names are its modules
|
|
1377
|
+
for s, sub in subs.items():
|
|
1378
|
+
if sub:
|
|
1379
|
+
name, _, alias = s.partition(" as ")
|
|
1380
|
+
bind(fid, imp.src_id, alias or name, (sub, None))
|
|
1381
|
+
self.import_targets[fid].add(sub)
|
|
1382
|
+
if (fid, sub) not in seen and sub != fid:
|
|
1383
|
+
seen.add((fid, sub))
|
|
1384
|
+
self.edges.append(Edge("imports", fid, sub, "exact", {"symbols": imp.symbols}))
|
|
1385
|
+
elif target:
|
|
1386
|
+
if imp.symbols:
|
|
1387
|
+
for s in imp.symbols:
|
|
1388
|
+
name, _, alias = s.partition(" as ")
|
|
1389
|
+
if name == "*":
|
|
1390
|
+
self.star_exports[fid].append(target)
|
|
1391
|
+
continue
|
|
1392
|
+
sub = self._py_module(fid, f"{imp.target}.{name}") if lang == "python" else None
|
|
1393
|
+
bind(fid, imp.src_id, alias or name, (sub, None) if sub else (target, name))
|
|
1394
|
+
elif imp.alias or lang == "python":
|
|
1395
|
+
bind(fid, imp.src_id, imp.alias or imp.target, (target, None))
|
|
1396
|
+
self.import_targets[fid].add(target)
|
|
1397
|
+
if (fid, target) not in seen and target != fid:
|
|
1398
|
+
seen.add((fid, target))
|
|
1399
|
+
self.edges.append(Edge("imports", fid, target, "heuristic" if on_path else "exact",
|
|
1400
|
+
{"symbols": imp.symbols, **({"found_by": "sys.path"} if on_path else {})}))
|
|
1401
|
+
elif lang == "python":
|
|
1402
|
+
top = imp.target.split(".")[0] or imp.target
|
|
1403
|
+
xid = self._external("python", top,
|
|
1404
|
+
{"category": "stdlib" if top in stdlib else "package"}, _repo_of(fid))
|
|
1405
|
+
if top: # `import sqlite3`: sqlite3.connect() is not ours to resolve
|
|
1406
|
+
self.outside_imports[fid].update(
|
|
1407
|
+
[(s.partition(" as ")[2] or s) for s in imp.symbols] if imp.symbols else [imp.alias or top])
|
|
1408
|
+
if (fid, xid) not in seen:
|
|
1409
|
+
seen.add((fid, xid))
|
|
1410
|
+
self.edges.append(Edge("imports", fid, xid, "exact", {"symbols": imp.symbols}))
|
|
1411
|
+
elif lang != "typescript" and not imp.target.startswith("."):
|
|
1412
|
+
# Go, Java, Rust...: the module or package path as written (go:example.com/app/store), not npm's.
|
|
1413
|
+
xid = self._external(lang, imp.target.strip("\"'<>` "), {"category": "package"}, _repo_of(fid))
|
|
1414
|
+
if (fid, xid) not in seen:
|
|
1415
|
+
seen.add((fid, xid))
|
|
1416
|
+
self.edges.append(Edge("imports", fid, xid, "exact", {"symbols": imp.symbols}))
|
|
1417
|
+
elif not imp.target.startswith("."):
|
|
1418
|
+
spec = imp.target[5:] if imp.target.startswith("node:") else imp.target
|
|
1419
|
+
parts = spec.split("/")
|
|
1420
|
+
top = "/".join(parts[:2]) if spec.startswith("@") else parts[0]
|
|
1421
|
+
builtin = imp.target.startswith("node:") or top in NODE_BUILTINS
|
|
1422
|
+
xid = self._external("npm", top, {"category": "stdlib" if builtin else "package"}, _repo_of(fid))
|
|
1423
|
+
for s in imp.symbols: # a name from outside: calls to it are not ours to resolve
|
|
1424
|
+
self.outside_imports[fid].add(s.partition(" as ")[2] or s)
|
|
1425
|
+
if imp.alias:
|
|
1426
|
+
self.outside_imports[fid].add(imp.alias)
|
|
1427
|
+
if (fid, xid) not in seen:
|
|
1428
|
+
seen.add((fid, xid))
|
|
1429
|
+
self.edges.append(Edge("imports", fid, xid, "exact", {"symbols": imp.symbols}))
|
|
1430
|
+
|
|
1431
|
+
def _script_entries(self) -> None:
|
|
1432
|
+
"""A TypeScript file that does work at its top level and that nothing imports is a program:
|
|
1433
|
+
a server's start file, a build script. Tests and config files are not."""
|
|
1434
|
+
imported = {t for targets in self.import_targets.values() for t in targets}
|
|
1435
|
+
for fid, res in self.results.items():
|
|
1436
|
+
if self.file_lang[fid] != "typescript" or fid in imported:
|
|
1437
|
+
continue
|
|
1438
|
+
path = self.nodes[fid].path
|
|
1439
|
+
base = path.rsplit("/", 1)[-1]
|
|
1440
|
+
top = f"{_repo_of(fid)}:typescript:{self._modpath(fid)}.<module>"
|
|
1441
|
+
if top not in self.nodes or top + "#entry" in self.nodes or re.search(r"\.(test|spec|bench|config|setup|d)\.", base) \
|
|
1442
|
+
or "/test/" in "/" + path or "/__tests__/" in "/" + path or any(n.kind == "test" for n in res.nodes):
|
|
1443
|
+
continue
|
|
1444
|
+
calls = [c for c in self._open(fid, res).calls if c.src_id == top and not c.ref]
|
|
1445
|
+
if len(calls) < 1:
|
|
1446
|
+
continue
|
|
1447
|
+
ep = Node(id=top + "#entry", kind="entry_point", name=path, parent_id=fid, language="typescript", path=path,
|
|
1448
|
+
span_start=1, span_end=self.nodes[fid].span_end, attrs={"trigger": "cli", "address": path, "found_by": "nothing imports it"})
|
|
1449
|
+
self._add(ep)
|
|
1450
|
+
self.file_of[ep.id] = fid
|
|
1451
|
+
self.edges.append(Edge("exposes", ep.id, top))
|
|
1452
|
+
|
|
1453
|
+
def _attach_methods(self) -> None:
|
|
1454
|
+
"""A method written outside its type's body (Go's func (t *T) Run(), Rust's impl T { ... } in another
|
|
1455
|
+
file) belongs to the type of that name, looked for in the same directory first."""
|
|
1456
|
+
types: dict[tuple, list[str]] = defaultdict(list)
|
|
1457
|
+
for n in self.nodes.values():
|
|
1458
|
+
if n.kind == "type":
|
|
1459
|
+
types[(n.language, n.name)].append(n.id)
|
|
1460
|
+
for n in list(self.nodes.values()):
|
|
1461
|
+
owner = n.attrs.get("owner_name") if n.kind == "callable" else None
|
|
1462
|
+
if not owner or (n.parent_id in self.nodes and self.nodes[n.parent_id].kind == "type"):
|
|
1463
|
+
continue
|
|
1464
|
+
cands = types.get((n.language, owner), [])
|
|
1465
|
+
here = (_repo_of(n.id), (n.path or "").rpartition("/")[0])
|
|
1466
|
+
near = [t for t in cands if (_repo_of(t), (self.nodes[t].path or "").rpartition("/")[0]) == here] or cands
|
|
1467
|
+
if len(near) == 1:
|
|
1468
|
+
n.parent_id = near[0]
|
|
1469
|
+
n.attrs["type_id"] = near[0]
|
|
1470
|
+
fid = f"{_repo_of(n.id)}:file:{n.path}"
|
|
1471
|
+
for res in (self.results.get(fid),):
|
|
1472
|
+
for c in (self._open(fid, res).calls if res else ()):
|
|
1473
|
+
if c.src_id == n.id and c.enclosing_type is None:
|
|
1474
|
+
c.enclosing_type = near[0]
|
|
1475
|
+
if self._opened is not None:
|
|
1476
|
+
self._opened[2] = True # packed again when it is let go
|
|
1477
|
+
|
|
1478
|
+
def _go_modules(self) -> None:
|
|
1479
|
+
"""Go modules of the workspace: each go.mod's `module` path, at the go.mod's directory (a repository may
|
|
1480
|
+
hold several, a go.work tying them), and its `replace` directives that point at a local directory."""
|
|
1481
|
+
self.go_mods: dict[str, tuple] = {} # module path -> (repo, directory in that repo)
|
|
1482
|
+
replaces = []
|
|
1483
|
+
for repo, files in self.files_of.items():
|
|
1484
|
+
for f in files:
|
|
1485
|
+
if f.rsplit("/", 1)[-1] != "go.mod":
|
|
1486
|
+
continue
|
|
1487
|
+
try:
|
|
1488
|
+
text = (self.repos[repo] / f).read_text(encoding="utf-8", errors="replace")
|
|
1489
|
+
except OSError:
|
|
1490
|
+
continue
|
|
1491
|
+
here = f.rpartition("/")[0]
|
|
1492
|
+
text = re.sub(r"//[^\n]*", "", text)
|
|
1493
|
+
m = re.search(r"^\s*module\s+\"?([^\s\"]+)", text, re.M)
|
|
1494
|
+
if m:
|
|
1495
|
+
self.go_mods.setdefault(m.group(1), (repo, here))
|
|
1496
|
+
# replace example.com/shared => ../shared, alone or in a replace ( ... ) block
|
|
1497
|
+
for r in re.finditer(r"^\s*(?:replace\s+)?([^\s(]+)(?:\s+v[^\s]+)?\s+=>\s+(\.{1,2}/[^\s]*|\.{1,2})\s*$",
|
|
1498
|
+
text, re.M):
|
|
1499
|
+
local = posixpath.normpath(posixpath.join(here, r.group(2)))
|
|
1500
|
+
if not local.startswith(".."):
|
|
1501
|
+
replaces.append((r.group(1), (repo, "" if local == "." else local)))
|
|
1502
|
+
for path, at in replaces:
|
|
1503
|
+
self.go_mods.setdefault(path, at)
|
|
1504
|
+
|
|
1505
|
+
def _go_import(self, fid: str, target: str) -> Optional[list]:
|
|
1506
|
+
"""The files of the package a Go import names, when it is under a module of the workspace: example.com/app/store
|
|
1507
|
+
with `module example.com/app` is the directory store/ beside that go.mod. None for a path under no module."""
|
|
1508
|
+
t = target.strip().strip('"`')
|
|
1509
|
+
mod = max((m for m in getattr(self, "go_mods", {}) if t == m or t.startswith(m + "/")), key=len, default=None)
|
|
1510
|
+
if mod is None:
|
|
1511
|
+
return None
|
|
1512
|
+
repo, base = self.go_mods[mod]
|
|
1513
|
+
d = posixpath.join(base, t[len(mod):].lstrip("/")).strip("/") if t != mod else base
|
|
1514
|
+
return sorted(f for f in self.results if _repo_of(f) == repo and self.file_lang[f] == "go"
|
|
1515
|
+
and self.nodes[f].path.rpartition("/")[0] == d)
|
|
1516
|
+
|
|
1517
|
+
def _find_module(self, fid: str, target: str) -> Optional[str]:
|
|
1518
|
+
if self.file_lang[fid] == "go":
|
|
1519
|
+
hits = self._go_import(fid, target)
|
|
1520
|
+
if hits is not None:
|
|
1521
|
+
for h in hits[1:]:
|
|
1522
|
+
self.import_targets[fid].add(h)
|
|
1523
|
+
return hits[0] if hits else None
|
|
1524
|
+
if getattr(BY_LANGUAGE.get(self.file_lang[fid]), "GENERIC", False):
|
|
1525
|
+
hits = self._generic_import(fid, target)
|
|
1526
|
+
for h in hits[1:]:
|
|
1527
|
+
self.import_targets[fid].add(h)
|
|
1528
|
+
return hits[0] if hits else None
|
|
1529
|
+
if self.file_lang[fid] == "python":
|
|
1530
|
+
return self._py_module(fid, target)
|
|
1531
|
+
hit = self._path_module(target, _repo_of(fid))
|
|
1532
|
+
if hit is None and self.file_lang[fid] == "typescript" and not target.startswith("."):
|
|
1533
|
+
hit = self._ts_alias(fid, target)
|
|
1534
|
+
return hit
|
|
1535
|
+
|
|
1536
|
+
def _ts_alias(self, fid: str, target: str) -> Optional[str]:
|
|
1537
|
+
"""An import through compilerOptions.paths of the nearest tsconfig.json or jsconfig.json above the file
|
|
1538
|
+
(`@ui/*` -> `src/*`), resolved against its baseUrl, or its own directory when it has none. `extends` is not
|
|
1539
|
+
followed: a tsconfig with no paths of its own lets the import fall to the next one up, as one that extends
|
|
1540
|
+
a shared base mostly does."""
|
|
1541
|
+
repo = _repo_of(fid)
|
|
1542
|
+
configs = self.__dict__.setdefault("_ts_configs", {})
|
|
1543
|
+
if repo not in configs:
|
|
1544
|
+
configs[repo] = found = {}
|
|
1545
|
+
for f in self.files_of.get(repo, ()):
|
|
1546
|
+
name = f.rsplit("/", 1)[-1]
|
|
1547
|
+
if not (name in ("tsconfig.json", "jsconfig.json") or re.fullmatch(r"tsconfig\.[\w.-]+\.json", name)):
|
|
1548
|
+
continue
|
|
1549
|
+
opts = (_jsonc(self.repos[repo] / f) or {}).get("compilerOptions")
|
|
1550
|
+
paths = opts.get("paths") if isinstance(opts, dict) else None
|
|
1551
|
+
if isinstance(paths, dict):
|
|
1552
|
+
here = f.rpartition("/")[0]
|
|
1553
|
+
base = posixpath.normpath(posixpath.join(here, str(opts.get("baseUrl") or ".")))
|
|
1554
|
+
found.setdefault(here, []).append(("" if base == "." else base, paths))
|
|
1555
|
+
here = self.nodes[fid].path.rpartition("/")[0]
|
|
1556
|
+
while True:
|
|
1557
|
+
for base, paths in configs[repo].get(here, ()):
|
|
1558
|
+
fits = []
|
|
1559
|
+
for key, subs in paths.items():
|
|
1560
|
+
pre, star, post = key.partition("*")
|
|
1561
|
+
if (target == key) if not star else (target.startswith(pre) and target.endswith(post)
|
|
1562
|
+
and len(target) >= len(pre) + len(post)):
|
|
1563
|
+
fits.append((len(pre), target[len(pre):len(target) - len(post)] if star else "", subs))
|
|
1564
|
+
for _, middle, subs in sorted(fits, key=lambda x: -x[0]): # the longest prefix wins, as in tsc
|
|
1565
|
+
for sub in subs if isinstance(subs, list) else [subs]:
|
|
1566
|
+
if isinstance(sub, str):
|
|
1567
|
+
path = posixpath.normpath(posixpath.join(base, sub.replace("*", middle, 1)))
|
|
1568
|
+
hit = self._path_module("./" + path, repo)
|
|
1569
|
+
if hit:
|
|
1570
|
+
return hit
|
|
1571
|
+
if fits:
|
|
1572
|
+
return None
|
|
1573
|
+
if not here:
|
|
1574
|
+
return None
|
|
1575
|
+
here = here.rpartition("/")[0]
|
|
1576
|
+
|
|
1577
|
+
def _generic_import(self, fid: str, target: str) -> list[str]:
|
|
1578
|
+
"""Files an import names, whatever the language writes: a relative path (./x, ../x.h), a dotted or ::
|
|
1579
|
+
path (com.foo.Bar, crate::a::b), or a package path ("github.com/x/y/pkg" names a directory)."""
|
|
1580
|
+
if not hasattr(self, "_suffixes"):
|
|
1581
|
+
self._suffixes: dict[str, list[str]] = defaultdict(list)
|
|
1582
|
+
self._dirs: dict[str, list[str]] = defaultdict(list)
|
|
1583
|
+
for f in self.results:
|
|
1584
|
+
path = self.nodes[f].path
|
|
1585
|
+
stem = re.sub(r"\.[A-Za-z0-9]+$", "", path)
|
|
1586
|
+
parts = stem.split("/")
|
|
1587
|
+
for i in range(len(parts)):
|
|
1588
|
+
self._suffixes["/".join(parts[i:])].append(f)
|
|
1589
|
+
if parts[-1] in ("mod", "index", "__init__", "lib") and i < len(parts) - 1:
|
|
1590
|
+
self._suffixes["/".join(parts[i:-1])].append(f)
|
|
1591
|
+
d = parts[:-1]
|
|
1592
|
+
for i in range(len(d)):
|
|
1593
|
+
self._dirs["/".join(d[i:])].append(f)
|
|
1594
|
+
lang, repo = self.file_lang[fid], _repo_of(fid)
|
|
1595
|
+
here = self.nodes[fid].path.rpartition("/")[0]
|
|
1596
|
+
t = target.strip().strip("\"'<>`")
|
|
1597
|
+
if not t:
|
|
1598
|
+
return []
|
|
1599
|
+
m = re.fullmatch(r"(\.+)([\w.]*)", t)
|
|
1600
|
+
if m: # Python's relative import: one dot is this package, each further dot one level up
|
|
1601
|
+
base = here
|
|
1602
|
+
for _ in range(len(m.group(1)) - 1):
|
|
1603
|
+
base = base.rpartition("/")[0]
|
|
1604
|
+
t = "./" + posixpath.join(posixpath.relpath(base or ".", here or "."), m.group(2).replace(".", "/")) if m.group(2) else "./"
|
|
1605
|
+
if not m.group(2):
|
|
1606
|
+
return [f for f in self._suffixes.get(posixpath.join(base, "__init__") if base else "__init__", []) if _repo_of(f) == repo][:1]
|
|
1607
|
+
if t.startswith("."):
|
|
1608
|
+
joined = posixpath.normpath(posixpath.join(here, t))
|
|
1609
|
+
key = re.sub(r"\.[A-Za-z0-9]+$", "", joined)
|
|
1610
|
+
return [f for f in self._suffixes.get(key, []) if self.nodes[f].path.startswith(key) and _repo_of(f) == repo] or self._dirs.get(key, [])[:0]
|
|
1611
|
+
t = re.sub(r"^(crate|self|super)::", "", t)
|
|
1612
|
+
key = re.sub(r"(::|\.|\\)", "/", t) if not "/" in t else re.sub(r"\.[A-Za-z0-9]+$", "", t)
|
|
1613
|
+
key = key.rstrip("/*").strip("/")
|
|
1614
|
+
def find(accept):
|
|
1615
|
+
same_lang = lambda fs, k: [f for f in fs if self.file_lang[f] == lang and accept(f, k)]
|
|
1616
|
+
for k in (key, key.rsplit("/", 1)[0] if "/" in key else None): # import a.b.C names a file a/b/C, or a symbol C in a/b
|
|
1617
|
+
if not k:
|
|
1618
|
+
continue
|
|
1619
|
+
hits = same_lang(self._suffixes.get(k, []), k)
|
|
1620
|
+
if hits:
|
|
1621
|
+
return hits[:8] if len(hits) <= 8 else []
|
|
1622
|
+
hits = same_lang(self._dirs.get(k, []), k) # a package or module directory
|
|
1623
|
+
if hits and len({self.nodes[h].path.rpartition("/")[0] for h in hits}) == 1:
|
|
1624
|
+
return hits[:40]
|
|
1625
|
+
return []
|
|
1626
|
+
own = find(lambda f, k: _repo_of(f) == repo)
|
|
1627
|
+
if own or len(self.repos) == 1:
|
|
1628
|
+
return own
|
|
1629
|
+
# Another repository of the workspace: only a name that starts at its root or at a source root there
|
|
1630
|
+
# (werkzeug/routing in src/werkzeug/routing.py), not any file that happens to end the same way (json).
|
|
1631
|
+
return find(lambda f, k: _repo_of(f) != repo and self._rooted(self.nodes[f].path, k))
|
|
1632
|
+
|
|
1633
|
+
SOURCE_ROOTS = {"src", "lib", "source", "sources", "main", "java", "kotlin", "scala", "pkg", "include", "app"}
|
|
1634
|
+
|
|
1635
|
+
def _rooted(self, path: str, key: str) -> bool:
|
|
1636
|
+
"""Whether `key` names `path` from the top of its repository or of a source root in it."""
|
|
1637
|
+
stem = re.sub(r"\.[A-Za-z0-9]+$", "", path)
|
|
1638
|
+
for tail in ("", "/mod", "/index", "/__init__", "/lib"):
|
|
1639
|
+
for cand in (stem, stem.rpartition("/")[0]): # the file itself, or its directory (a package)
|
|
1640
|
+
if tail and not cand.endswith(tail):
|
|
1641
|
+
continue
|
|
1642
|
+
cand = cand[: len(cand) - len(tail)] if tail else cand
|
|
1643
|
+
if cand == key or cand.endswith("/" + key):
|
|
1644
|
+
prefix = cand[: len(cand) - len(key)].strip("/")
|
|
1645
|
+
if not prefix or all(p in self.SOURCE_ROOTS for p in prefix.split("/")):
|
|
1646
|
+
return True
|
|
1647
|
+
return False
|
|
1648
|
+
|
|
1649
|
+
def _path_module(self, target: str, repo: Optional[str] = None) -> Optional[str]:
|
|
1650
|
+
"""A TypeScript import: ./path from the repo root (the adapter resolved it), or a workspace package,
|
|
1651
|
+
which may live in another repository of the workspace."""
|
|
1652
|
+
def at(path, repo=repo or self.repo):
|
|
1653
|
+
# only a leading ./ goes: lstrip("./") also took the dot off a directory such as .storybook
|
|
1654
|
+
path = re.sub(r"\.(d\.[cm]?ts|[cm]?[jt]sx?)$", "", path[2:] if path.startswith("./") else path)
|
|
1655
|
+
return self.path_modules[repo].get(path) or self.path_modules[repo].get(path + "/index")
|
|
1656
|
+
if target.startswith("."):
|
|
1657
|
+
return at(target[2:] if target.startswith("./") else target)
|
|
1658
|
+
name = max((p for p in self.packages if target == p or target.startswith(p + "/")), key=len, default=None)
|
|
1659
|
+
if name is None:
|
|
1660
|
+
return None
|
|
1661
|
+
pkg, sub = self.packages[name], target[len(name):].lstrip("/")
|
|
1662
|
+
exports = pkg["exports"]
|
|
1663
|
+
entry = exports.get("./" + sub if sub else ".") if isinstance(exports, dict) else (exports if not sub else None)
|
|
1664
|
+
while isinstance(entry, dict): # {"import": ..., "types": ..., "default": ...}
|
|
1665
|
+
entry = next((entry[k] for k in ("source", "import", "default", "types", "require") if k in entry), None)
|
|
1666
|
+
tries = [entry] if isinstance(entry, str) else []
|
|
1667
|
+
tries += [f"{sub}", f"src/{sub}"] if sub else [pkg["main"], "src/index", "index"]
|
|
1668
|
+
for t in tries:
|
|
1669
|
+
if t:
|
|
1670
|
+
hit = at(posixpath.normpath(posixpath.join(pkg["dir"], t)), pkg["repo"])
|
|
1671
|
+
if hit:
|
|
1672
|
+
return hit
|
|
1673
|
+
return None
|
|
1674
|
+
|
|
1675
|
+
def _py_module(self, fid: str, target: str) -> Optional[str]:
|
|
1676
|
+
repo = _repo_of(fid)
|
|
1677
|
+
mods = self.py_modules[repo]
|
|
1678
|
+
if target in mods:
|
|
1679
|
+
return mods[target]
|
|
1680
|
+
own = module_path(self.nodes[fid].path)
|
|
1681
|
+
pkg = own.rsplit(".", 1)[0] if "." in own else ""
|
|
1682
|
+
sibling = f"{pkg}.{target}" if pkg else target
|
|
1683
|
+
if sibling in mods: # script-style import of a file in the same directory
|
|
1684
|
+
return mods[sibling]
|
|
1685
|
+
hit = self.py_importable.get(target) # a package of another repository in the workspace
|
|
1686
|
+
return hit if hit and _repo_of(hit) != repo else None
|
|
1687
|
+
|
|
1688
|
+
def _py_on_path(self, fid: str, target: str) -> Optional[str]:
|
|
1689
|
+
"""`sys.path.insert(0, <dir>)` then `import validate`: a script or a test that puts a directory on the path
|
|
1690
|
+
imports a file there by its bare name. When the importing file, or a conftest.py above it, changes sys.path,
|
|
1691
|
+
an import nothing else resolves is taken to be the repository's one module of that name (or the one in a
|
|
1692
|
+
directory above the importer, when there are several). A heuristic: which directory is added is not worked
|
|
1693
|
+
out, so the import edge says so."""
|
|
1694
|
+
if target.startswith(".") or not self._edits_sys_path(fid):
|
|
1695
|
+
return None
|
|
1696
|
+
repo = _repo_of(fid)
|
|
1697
|
+
tail = "." + target
|
|
1698
|
+
cands = sorted({f for m, f in self.py_modules[repo].items() if (m == target or m.endswith(tail)) and f != fid})
|
|
1699
|
+
if len(cands) > 1:
|
|
1700
|
+
here = self.nodes[fid].path.rpartition("/")[0]
|
|
1701
|
+
cands = [f for f in cands if (here + "/").startswith(self.nodes[f].path.rpartition("/")[0].rstrip("/") + "/")
|
|
1702
|
+
or "/" not in self.nodes[f].path]
|
|
1703
|
+
return cands[0] if len(cands) == 1 else None
|
|
1704
|
+
|
|
1705
|
+
def _edits_sys_path(self, fid: str) -> bool:
|
|
1706
|
+
cache = self.__dict__.setdefault("_sys_path_files", {})
|
|
1707
|
+
if fid not in cache:
|
|
1708
|
+
repo, path = _repo_of(fid), self.nodes[fid].path
|
|
1709
|
+
parts = path.split("/")
|
|
1710
|
+
files = [path] + ["/".join(parts[:i] + ["conftest.py"]) for i in range(len(parts) - 1, -1, -1)]
|
|
1711
|
+
hit = False
|
|
1712
|
+
for p in dict.fromkeys(files):
|
|
1713
|
+
if p != path and module_path(p) not in self.py_modules[repo]:
|
|
1714
|
+
continue
|
|
1715
|
+
try:
|
|
1716
|
+
hit = b"sys.path" in read_source(self.repos[repo] / p)
|
|
1717
|
+
except OSError:
|
|
1718
|
+
hit = False
|
|
1719
|
+
if hit:
|
|
1720
|
+
break
|
|
1721
|
+
cache[fid] = hit
|
|
1722
|
+
return cache[fid]
|
|
1723
|
+
|
|
1724
|
+
def _modpath(self, fid: str) -> str:
|
|
1725
|
+
adapter = BY_LANGUAGE[self.file_lang[fid]]
|
|
1726
|
+
if fid in self.keep_ext:
|
|
1727
|
+
return adapter.module_path(self.nodes[fid].path, True)
|
|
1728
|
+
return adapter.module_path(self.nodes[fid].path)
|
|
1729
|
+
|
|
1730
|
+
def _py_export(self, target_fid: str, symbol: str, depth: int = 0) -> Optional[str]:
|
|
1731
|
+
"""The node a module exposes under a name, following re-exports (`from .app import Flask`,
|
|
1732
|
+
`export { a } from "./b"`, `export * from "./c"`)."""
|
|
1733
|
+
if target_fid not in self.file_lang:
|
|
1734
|
+
return None
|
|
1735
|
+
if symbol == "default" and target_fid in self.default_export:
|
|
1736
|
+
return self.default_export[target_fid]
|
|
1737
|
+
cand = f"{_repo_of(target_fid)}:{self.file_lang[target_fid]}:{self._modpath(target_fid)}.{symbol}"
|
|
1738
|
+
if cand in self.nodes:
|
|
1739
|
+
return cand
|
|
1740
|
+
if depth < 6 and symbol in self.py_names.get(target_fid, {}):
|
|
1741
|
+
nxt, sub = self.py_names[target_fid][symbol]
|
|
1742
|
+
if sub is not None:
|
|
1743
|
+
return self._py_export(nxt, sub, depth + 1)
|
|
1744
|
+
if depth < 6:
|
|
1745
|
+
for nxt in self.star_exports.get(target_fid, ()):
|
|
1746
|
+
hit = self._py_export(nxt, symbol, depth + 1)
|
|
1747
|
+
if hit:
|
|
1748
|
+
return hit
|
|
1749
|
+
return None
|
|
1750
|
+
|
|
1751
|
+
def _export_module(self, target_fid: str, symbol: str, depth: int = 0) -> Optional[str]:
|
|
1752
|
+
"""The file a module exposes whole under a name (`export * as ns from './format'`, `import util as u`),
|
|
1753
|
+
following re-exports, or None."""
|
|
1754
|
+
if depth > 6:
|
|
1755
|
+
return None
|
|
1756
|
+
bound = self.py_names.get(target_fid, {}).get(symbol)
|
|
1757
|
+
if bound:
|
|
1758
|
+
nxt, sub = bound
|
|
1759
|
+
return nxt if sub is None else self._export_module(nxt, sub, depth + 1)
|
|
1760
|
+
for nxt in self.star_exports.get(target_fid, ()):
|
|
1761
|
+
hit = self._export_module(nxt, symbol, depth + 1)
|
|
1762
|
+
if hit:
|
|
1763
|
+
return hit
|
|
1764
|
+
return None
|
|
1765
|
+
|
|
1766
|
+
def _py_name(self, fid: str, src: str, name: str) -> tuple:
|
|
1767
|
+
"""What a name imported in a file means inside one of its functions: an import in the function or an
|
|
1768
|
+
enclosing one first, then the file's. (None, None) when it cannot be told."""
|
|
1769
|
+
cur = src
|
|
1770
|
+
while cur in self.nodes and self.nodes[cur].kind == "callable":
|
|
1771
|
+
if (cur, name) in self.py_local:
|
|
1772
|
+
return self.py_local[(cur, name)]
|
|
1773
|
+
cur = self.nodes[cur].parent_id
|
|
1774
|
+
return self.py_names[fid].get(name, (None, None))
|
|
1775
|
+
|
|
1776
|
+
def _py_bound(self, fid: str, src: Optional[str], name: str) -> bool:
|
|
1777
|
+
"""Does an import bind this name where src runs: in src, a function around it, or the file?"""
|
|
1778
|
+
cur = src
|
|
1779
|
+
while cur in self.nodes and self.nodes[cur].kind == "callable":
|
|
1780
|
+
if (cur, name) in self.py_local:
|
|
1781
|
+
return True
|
|
1782
|
+
cur = self.nodes[cur].parent_id
|
|
1783
|
+
return name in self.py_names[fid]
|
|
1784
|
+
|
|
1785
|
+
def _py_var(self, fid: str, name: str, depth: int = 0) -> Optional[str]:
|
|
1786
|
+
"""The type of a module-level variable (`g: Globals = ...`), found through imports and re-exports."""
|
|
1787
|
+
tn = self.py_vars.get(fid, {}).get(name)
|
|
1788
|
+
if tn:
|
|
1789
|
+
return self._type("python", tn, None, file=fid)
|
|
1790
|
+
if depth < 6 and name in self.py_names.get(fid, {}):
|
|
1791
|
+
nxt, sub = self.py_names[fid][name]
|
|
1792
|
+
if sub is not None:
|
|
1793
|
+
return self._py_var(nxt, sub, depth + 1)
|
|
1794
|
+
if depth < 6:
|
|
1795
|
+
for nxt in self.star_exports.get(fid, ()):
|
|
1796
|
+
hit = self._py_var(nxt, name, depth + 1)
|
|
1797
|
+
if hit:
|
|
1798
|
+
return hit
|
|
1799
|
+
return None
|
|
1800
|
+
|
|
1801
|
+
def _attr_type(self, lang: str, inner: CallSite) -> Optional[tuple]:
|
|
1802
|
+
"""The type of an attribute read (`app.config`, `flask.g`) used as a receiver: the declared type of the
|
|
1803
|
+
field, of a property's result, or of a module's variable. None when it cannot be told."""
|
|
1804
|
+
fid = self.file_of.get(inner.src_id)
|
|
1805
|
+
if lang == "python" and fid and inner.receiver_type is None and inner.chain is None and self._py_bound(fid, inner.src_id, inner.receiver or ""):
|
|
1806
|
+
target, symbol = self._py_name(fid, inner.src_id, inner.receiver)
|
|
1807
|
+
if target and symbol is None:
|
|
1808
|
+
tid = self._py_var(target, inner.name)
|
|
1809
|
+
return (tid, True) if tid else None
|
|
1810
|
+
tid, _ = self._receiver_type(lang, inner)
|
|
1811
|
+
for t in self._chain(tid):
|
|
1812
|
+
tn = self.field_type.get(t, {}).get(inner.name)
|
|
1813
|
+
if tn:
|
|
1814
|
+
got = self._type(lang, tn, f"{t}.{inner.name}", file=self.file_of.get(t))
|
|
1815
|
+
return (got, True) if got else None
|
|
1816
|
+
getters = [m for m in self.members.get(t, {}).get(inner.name, ()) if m.attrs.get("returns")
|
|
1817
|
+
and any("property" in d for d in m.attrs.get("decorators") or [])]
|
|
1818
|
+
if getters:
|
|
1819
|
+
got = self._type(lang, getters[0].attrs["returns"], getters[0].id)
|
|
1820
|
+
return (got, True) if got else None
|
|
1821
|
+
return None
|
|
1822
|
+
|
|
1823
|
+
def _py_fixtures(self) -> None:
|
|
1824
|
+
"""pytest passes a test each fixture named by its parameters. Link them, and give the
|
|
1825
|
+
parameter the fixture's return type so calls on it can be resolved."""
|
|
1826
|
+
self.py_param_type: dict[tuple, str] = {}
|
|
1827
|
+
by_file: dict[str, dict[str, Node]] = defaultdict(dict)
|
|
1828
|
+
for n in self.nodes.values():
|
|
1829
|
+
if n.language == "python" and n.kind == "callable" and n.attrs.get("is_fixture"):
|
|
1830
|
+
by_file[self.file_of[n.id]][n.name] = n
|
|
1831
|
+
if not by_file:
|
|
1832
|
+
return
|
|
1833
|
+
conftests = {(_repo_of(f), self.nodes[f].path.rsplit("/", 1)[0] if "/" in self.nodes[f].path else ""): f
|
|
1834
|
+
for f in by_file if self.nodes[f].path.rsplit("/", 1)[-1] == "conftest.py"}
|
|
1835
|
+
|
|
1836
|
+
def find(fid: str, name: str, skip: Optional[str] = None) -> Optional[Node]:
|
|
1837
|
+
hit = by_file.get(fid, {}).get(name)
|
|
1838
|
+
if hit is not None and hit.id != skip:
|
|
1839
|
+
return hit
|
|
1840
|
+
d = self.nodes[fid].path.rsplit("/", 1)[0] if "/" in self.nodes[fid].path else ""
|
|
1841
|
+
while True:
|
|
1842
|
+
hit = by_file.get(conftests.get((_repo_of(fid), d), ""), {}).get(name)
|
|
1843
|
+
if hit is not None and hit.id != skip:
|
|
1844
|
+
return hit
|
|
1845
|
+
if not d:
|
|
1846
|
+
return None
|
|
1847
|
+
d = d.rsplit("/", 1)[0] if "/" in d else ""
|
|
1848
|
+
memo: dict[str, Optional[str]] = {}
|
|
1849
|
+
|
|
1850
|
+
def returns(fx: Node, depth: int = 0) -> Optional[str]:
|
|
1851
|
+
if fx.id in memo or depth > 4:
|
|
1852
|
+
return memo.get(fx.id)
|
|
1853
|
+
memo[fx.id] = None
|
|
1854
|
+
out = None
|
|
1855
|
+
if fx.attrs.get("returns"):
|
|
1856
|
+
out = self._type("python", fx.attrs["returns"], fx.id)
|
|
1857
|
+
elif fx.attrs.get("returns_call"):
|
|
1858
|
+
recv, method = fx.attrs["returns_call"]
|
|
1859
|
+
dep = find(self.file_of[fx.id], recv, fx.id) if recv in (fx.attrs.get("params") or []) else None
|
|
1860
|
+
owner = returns(dep, depth + 1) if dep is not None else None
|
|
1861
|
+
for m in self._methods(owner, method, 0) if owner else []:
|
|
1862
|
+
if m.attrs.get("returns"):
|
|
1863
|
+
out = self._type("python", m.attrs["returns"], m.id)
|
|
1864
|
+
memo[fx.id] = out
|
|
1865
|
+
return out
|
|
1866
|
+
for n in list(self.nodes.values()):
|
|
1867
|
+
if n.language != "python" or n.kind != "callable" or not (n.attrs.get("is_test") or n.attrs.get("is_fixture")):
|
|
1868
|
+
continue
|
|
1869
|
+
for pname in n.attrs.get("params") or []:
|
|
1870
|
+
fx = find(self.file_of[n.id], pname, n.id)
|
|
1871
|
+
if fx is None:
|
|
1872
|
+
continue
|
|
1873
|
+
self.calls.append((n.id, fx.id, "fixture", "heuristic", n.span_start))
|
|
1874
|
+
tid = returns(fx)
|
|
1875
|
+
if tid:
|
|
1876
|
+
self.py_param_type[(n.id, pname)] = tid
|
|
1877
|
+
|
|
1878
|
+
# -- visibility ----------------------------------------------------------
|
|
1879
|
+
def _visible(self, fid: str) -> Optional[set]:
|
|
1880
|
+
"""Modules whose symbols a file can reference, or None when that is not knowable."""
|
|
1881
|
+
mod = self.nodes[fid].parent_id
|
|
1882
|
+
if self.file_lang[fid] in FILE_MODULE:
|
|
1883
|
+
if fid not in self._vis_cache:
|
|
1884
|
+
out, queue = {fid}, [t for t, _ in self.py_names[fid].values() if t] + list(self.star_exports.get(fid, ()))
|
|
1885
|
+
if self.file_lang[fid] == "python":
|
|
1886
|
+
queue += list(self.import_targets.get(fid, ())) # imports inside functions count too
|
|
1887
|
+
while queue: # a file re-exported with `export *` is seen through the file that re-exports it
|
|
1888
|
+
cur = queue.pop()
|
|
1889
|
+
if cur not in out:
|
|
1890
|
+
out.add(cur)
|
|
1891
|
+
queue.extend(self.star_exports.get(cur, ()))
|
|
1892
|
+
self._vis_cache[fid] = out
|
|
1893
|
+
return self._vis_cache[fid]
|
|
1894
|
+
if not (self.nodes[mod].attrs.get("marker") or "").endswith(".csproj"):
|
|
1895
|
+
return None # a loose .cs file: no project file says what it can see
|
|
1896
|
+
if mod not in self._vis_cache:
|
|
1897
|
+
seen, queue = {mod}, [mod]
|
|
1898
|
+
while queue:
|
|
1899
|
+
cur = queue.pop()
|
|
1900
|
+
for nxt in self.project_refs.get(cur, ()):
|
|
1901
|
+
if nxt not in seen:
|
|
1902
|
+
seen.add(nxt)
|
|
1903
|
+
queue.append(nxt)
|
|
1904
|
+
self._vis_cache[mod] = seen
|
|
1905
|
+
return self._vis_cache[mod]
|
|
1906
|
+
|
|
1907
|
+
def _file_set(self, key: tuple) -> Optional[set]:
|
|
1908
|
+
"""A set kept for the file whose calls are being resolved. Calls are resolved a file at a time, so only the
|
|
1909
|
+
last few files' sets are kept: keeping one per file held millions of members on a large repository (every
|
|
1910
|
+
Go file sees each file of every package it imports). Import targets are settled before any of these is
|
|
1911
|
+
asked for, so a set made again is the same set."""
|
|
1912
|
+
return self._sets.get(key)
|
|
1913
|
+
|
|
1914
|
+
def _keep_set(self, key: tuple, value: set) -> set:
|
|
1915
|
+
if len(self._sets) >= 16:
|
|
1916
|
+
self._sets.clear()
|
|
1917
|
+
self._sets[key] = value
|
|
1918
|
+
return value
|
|
1919
|
+
|
|
1920
|
+
def _reach(self, fid: str) -> set:
|
|
1921
|
+
"""Files reachable through imports. A name found nowhere else in that set is a fair guess."""
|
|
1922
|
+
key = ("reach", fid)
|
|
1923
|
+
got = self._file_set(key)
|
|
1924
|
+
if got is None:
|
|
1925
|
+
out, queue = set(), [fid]
|
|
1926
|
+
while queue:
|
|
1927
|
+
cur = queue.pop()
|
|
1928
|
+
if cur in out:
|
|
1929
|
+
continue
|
|
1930
|
+
out.add(cur)
|
|
1931
|
+
queue.extend(t for t in self.import_targets.get(cur, ()) if t not in out)
|
|
1932
|
+
got = self._keep_set(key, out)
|
|
1933
|
+
return got
|
|
1934
|
+
|
|
1935
|
+
def _can_see(self, fid: str, node_id: str) -> bool:
|
|
1936
|
+
vis = self._visible(fid)
|
|
1937
|
+
if vis is None:
|
|
1938
|
+
return True
|
|
1939
|
+
target_file = self.file_of.get(node_id)
|
|
1940
|
+
if self.file_lang[fid] in FILE_MODULE:
|
|
1941
|
+
return target_file in vis
|
|
1942
|
+
return target_file is not None and self.nodes[target_file].parent_id in vis
|
|
1943
|
+
|
|
1944
|
+
# -- types ---------------------------------------------------------------
|
|
1945
|
+
def _type(self, lang: str, name: str, from_id: Optional[str], file: Optional[str] = None) -> Optional[str]:
|
|
1946
|
+
fid = file or self.file_of.get(from_id or "")
|
|
1947
|
+
arity = None
|
|
1948
|
+
if lang == "python" and "." in name:
|
|
1949
|
+
# flask.Flask: the class a module imported under that name exposes.
|
|
1950
|
+
head, _, rest = name.rpartition(".")
|
|
1951
|
+
mod = self._py_name(fid, from_id, head) if fid and "." not in head else None
|
|
1952
|
+
cand = self._py_export(mod[0], rest) if mod and mod[0] and mod[1] is None else None
|
|
1953
|
+
if cand and self.nodes[cand].kind == "type":
|
|
1954
|
+
return cand
|
|
1955
|
+
name = rest
|
|
1956
|
+
if lang == "csharp":
|
|
1957
|
+
name, tick, count = name.partition("`")
|
|
1958
|
+
arity = int(count) if tick and count.isdigit() else 0
|
|
1959
|
+
if fid:
|
|
1960
|
+
name = self.cs_alias[fid].get(name, name)
|
|
1961
|
+
if lang == "python" and from_id:
|
|
1962
|
+
cur = from_id # a class defined in the function itself or one around it
|
|
1963
|
+
while cur in self.nodes and self.nodes[cur].kind in ("callable", "test", "type"):
|
|
1964
|
+
hit = self.nodes.get(f"{cur}.{name}")
|
|
1965
|
+
if hit is not None and hit.kind == "type" and hit.id != from_id: # `class App(web.App)` is not its own base
|
|
1966
|
+
return hit.id
|
|
1967
|
+
cur = self.nodes[cur].parent_id
|
|
1968
|
+
if lang in FILE_MODULE and fid and (self._py_bound(fid, from_id, name) if lang == "python" else name in self.py_names[fid]):
|
|
1969
|
+
target, symbol = self._py_name(fid, from_id, name) if lang == "python" else self.py_names[fid][name]
|
|
1970
|
+
cand = self._py_export(target, symbol or name)
|
|
1971
|
+
if cand and self.nodes[cand].kind == "type":
|
|
1972
|
+
return cand
|
|
1973
|
+
if lang == "typescript" and fid:
|
|
1974
|
+
if name in self.outside_imports[fid]:
|
|
1975
|
+
return None
|
|
1976
|
+
own = f"{_repo_of(fid)}:{lang}:{self._modpath(fid)}.{name}"
|
|
1977
|
+
if own in self.nodes and self.nodes[own].kind == "type":
|
|
1978
|
+
return own
|
|
1979
|
+
cands = self.types_by_name.get((lang, name), [])
|
|
1980
|
+
if lang == "python":
|
|
1981
|
+
# A class defined inside a function is seen only there (found above), not by name from elsewhere.
|
|
1982
|
+
cands = [c for c in cands if self.nodes[self.nodes[c].parent_id].kind not in ("callable", "test")]
|
|
1983
|
+
if lang == "csharp" and fid:
|
|
1984
|
+
cands = [c for c in cands if self._can_see(fid, c)]
|
|
1985
|
+
if arity is not None and cands:
|
|
1986
|
+
# Foo and Foo<T> are different types. A plain name falls back to the generic one, because
|
|
1987
|
+
# some callers (a receiver's inferred type) do not carry the type arguments.
|
|
1988
|
+
exact = [c for c in cands if _arity(c) == arity]
|
|
1989
|
+
cands = exact or ([] if arity else cands)
|
|
1990
|
+
if not cands:
|
|
1991
|
+
return None
|
|
1992
|
+
if len(cands) == 1:
|
|
1993
|
+
return cands[0]
|
|
1994
|
+
if fid:
|
|
1995
|
+
same_file = [c for c in cands if self.file_of.get(c) == fid]
|
|
1996
|
+
if same_file:
|
|
1997
|
+
return same_file[0]
|
|
1998
|
+
if lang == "csharp":
|
|
1999
|
+
here = set(self.cs_usings[fid])
|
|
2000
|
+
for ns in self.results[fid].declares: # a namespace sees its parents
|
|
2001
|
+
parts = ns.split(".")
|
|
2002
|
+
here.update(".".join(parts[:i]) for i in range(1, len(parts) + 1))
|
|
2003
|
+
here.add("")
|
|
2004
|
+
visible = [c for c in cands if self.nodes[c].attrs.get("namespace") in here]
|
|
2005
|
+
if len(visible) >= 1:
|
|
2006
|
+
return visible[0]
|
|
2007
|
+
return None
|
|
2008
|
+
|
|
2009
|
+
def _resolve_types(self) -> None:
|
|
2010
|
+
seen = set()
|
|
2011
|
+
for fid, res in self.results.items():
|
|
2012
|
+
lang = self.file_lang[fid]
|
|
2013
|
+
adapter = BY_LANGUAGE[lang].NAME
|
|
2014
|
+
for ref in res.type_refs:
|
|
2015
|
+
for i, name in enumerate(ref.names):
|
|
2016
|
+
tid = self._type(lang, name, ref.src_id)
|
|
2017
|
+
if tid is None:
|
|
2018
|
+
self.stats[adapter]["type_refs_external"] += 1
|
|
2019
|
+
continue
|
|
2020
|
+
self.stats[adapter]["type_refs_resolved"] += 1
|
|
2021
|
+
if ref.role == "base":
|
|
2022
|
+
if i > 0:
|
|
2023
|
+
kind, attrs = "uses_type", {"role": "generic_arg"}
|
|
2024
|
+
else:
|
|
2025
|
+
target_iface = self.nodes[tid].attrs.get("native_kind") == "interface"
|
|
2026
|
+
src_iface = self.nodes[ref.src_id].attrs.get("native_kind") == "interface"
|
|
2027
|
+
kind, attrs = ("implements" if target_iface and not src_iface else "extends"), {}
|
|
2028
|
+
self.bases[ref.src_id].append(tid)
|
|
2029
|
+
elif ref.role == "instantiate":
|
|
2030
|
+
kind, attrs = ("instantiates" if i == 0 else "uses_type"), ({} if i == 0 else {"role": "generic_arg"})
|
|
2031
|
+
else:
|
|
2032
|
+
kind, attrs = "uses_type", {"role": ref.role if i == 0 else "generic_arg"}
|
|
2033
|
+
key = (kind, ref.src_id, tid, attrs.get("role"))
|
|
2034
|
+
if key in seen or ref.src_id == tid:
|
|
2035
|
+
continue
|
|
2036
|
+
seen.add(key)
|
|
2037
|
+
self.edges.append(Edge(kind, ref.src_id, tid, "heuristic", attrs))
|
|
2038
|
+
|
|
2039
|
+
# -- calls ---------------------------------------------------------------
|
|
2040
|
+
def _chain(self, type_id: Optional[str], enclosing: bool = True) -> list[str]:
|
|
2041
|
+
"""A type, its bases, and (unless `enclosing` is off) its enclosing types, nearest first. A name written bare
|
|
2042
|
+
inside a nested type can mean the enclosing type's static member; a call on a receiver (`this`, `self`, a
|
|
2043
|
+
typed variable) reaches only the receiver's own type and its bases."""
|
|
2044
|
+
out, queue = [], [type_id] if type_id else []
|
|
2045
|
+
while queue:
|
|
2046
|
+
t = queue.pop(0)
|
|
2047
|
+
if t in out or t not in self.nodes:
|
|
2048
|
+
continue
|
|
2049
|
+
out.append(t)
|
|
2050
|
+
queue.extend(self.bases.get(t, []))
|
|
2051
|
+
parent = self.nodes[t].parent_id
|
|
2052
|
+
if enclosing and parent in self.nodes and self.nodes[parent].kind == "type":
|
|
2053
|
+
queue.append(parent)
|
|
2054
|
+
return out
|
|
2055
|
+
|
|
2056
|
+
def _pick(self, cands: list[Node], argc: int, skip: int = 0) -> list[Node]:
|
|
2057
|
+
"""Overloads that fit a call. `skip` is 1 for an extension method, whose first parameter is the receiver."""
|
|
2058
|
+
if argc < 0:
|
|
2059
|
+
return list(cands) # a function named, not called: nothing to match arguments against
|
|
2060
|
+
argc += skip
|
|
2061
|
+
fit = [c for c in cands if c.attrs.get("argc_min", 0) <= argc <= c.attrs.get("argc_max", 99)]
|
|
2062
|
+
call = getattr(self, "_call", None)
|
|
2063
|
+
if len(fit) > 1 and call is not None:
|
|
2064
|
+
# Several overloads take this many arguments. Narrow by what the call site shows.
|
|
2065
|
+
if call.targs:
|
|
2066
|
+
fit = [c for c in fit if c.attrs.get("generic_arity") == call.targs] or fit
|
|
2067
|
+
hints = call.args
|
|
2068
|
+
|
|
2069
|
+
def possible(c: Node) -> bool:
|
|
2070
|
+
want = c.attrs.get("delegate_arity") or []
|
|
2071
|
+
for i, h in enumerate(hints, skip):
|
|
2072
|
+
if isinstance(h, int) and not isinstance(h, bool) and i < len(want):
|
|
2073
|
+
if want[i] == -2 or (want[i] >= 0 and want[i] != h):
|
|
2074
|
+
return False # a lambda passed where no delegate of that shape is taken
|
|
2075
|
+
elif isinstance(h, str) and i < len(want) and want[i] >= 0 and h in ("int", "double", "bool", "char", "string"):
|
|
2076
|
+
return False # a literal passed where a delegate is taken
|
|
2077
|
+
return True
|
|
2078
|
+
|
|
2079
|
+
def score(c: Node) -> int:
|
|
2080
|
+
want, types = c.attrs.get("delegate_arity") or [], c.attrs.get("param_types") or []
|
|
2081
|
+
n = 0
|
|
2082
|
+
for i, h in enumerate(hints, skip):
|
|
2083
|
+
if isinstance(h, int) and i < len(want) and want[i] == h:
|
|
2084
|
+
n += 1
|
|
2085
|
+
elif isinstance(h, str) and i < len(types) and types[i].split("`")[0] == h:
|
|
2086
|
+
n += 1
|
|
2087
|
+
return n
|
|
2088
|
+
fit = [c for c in fit if possible(c)] or fit
|
|
2089
|
+
best = max(score(c) for c in fit)
|
|
2090
|
+
fit = [c for c in fit if score(c) == best]
|
|
2091
|
+
return fit or cands
|
|
2092
|
+
|
|
2093
|
+
def _methods(self, type_id: Optional[str], name: str, argc: int, enclosing: bool = True) -> list[Node]:
|
|
2094
|
+
for t in self._chain(type_id, enclosing):
|
|
2095
|
+
cands = self.members.get(t, {}).get(name)
|
|
2096
|
+
if cands:
|
|
2097
|
+
return self._pick(cands, argc)
|
|
2098
|
+
return []
|
|
2099
|
+
|
|
2100
|
+
def _receiver_type(self, lang: str, call: CallSite) -> tuple[Optional[str], bool]:
|
|
2101
|
+
"""Returns (type id, known). known=True with None means an external type."""
|
|
2102
|
+
r = call.receiver
|
|
2103
|
+
if r in (None, "this"):
|
|
2104
|
+
return call.enclosing_type, call.enclosing_type is not None
|
|
2105
|
+
if r == "base":
|
|
2106
|
+
bases = self.bases.get(call.enclosing_type or "", [])
|
|
2107
|
+
return (bases[0] if bases else None), True
|
|
2108
|
+
if call.receiver_type:
|
|
2109
|
+
return self._type(lang, call.receiver_type, call.src_id), True
|
|
2110
|
+
if call.chain is not None and not call.receiver_type:
|
|
2111
|
+
got = self._returned_type(lang, call.chain)
|
|
2112
|
+
if got is not None:
|
|
2113
|
+
return got
|
|
2114
|
+
if r in ("?", "[]"):
|
|
2115
|
+
return None, False
|
|
2116
|
+
if r in getattr(BY_LANGUAGE[lang], "GLOBALS", ()) and not self._type(lang, r, call.src_id):
|
|
2117
|
+
return None, True # JSON.stringify, Math.max: the platform's, not ours
|
|
2118
|
+
if r.startswith("."):
|
|
2119
|
+
names = self.field_types_global.get((lang, r[1:]), set())
|
|
2120
|
+
ids = {self._type(lang, n, call.src_id) for n in names}
|
|
2121
|
+
ids.discard(None)
|
|
2122
|
+
if len(ids) == 1:
|
|
2123
|
+
return ids.pop(), True
|
|
2124
|
+
if names and not ids:
|
|
2125
|
+
return None, True # every field of that name has an external type
|
|
2126
|
+
return None, False
|
|
2127
|
+
if lang == "python":
|
|
2128
|
+
# A parameter filled by a pytest fixture, seen from the test or a function nested in it.
|
|
2129
|
+
cur, shadowed = call.src_id, False
|
|
2130
|
+
while cur in self.nodes and self.nodes[cur].kind == "callable":
|
|
2131
|
+
if (cur, r) in self.py_param_type:
|
|
2132
|
+
return self.py_param_type[(cur, r)], True
|
|
2133
|
+
if r in (self.nodes[cur].attrs.get("params") or []):
|
|
2134
|
+
shadowed = True
|
|
2135
|
+
break
|
|
2136
|
+
cur = self.nodes[cur].parent_id
|
|
2137
|
+
fid = self.file_of.get(call.src_id)
|
|
2138
|
+
if not shadowed and call.chain is None and fid and (r in self.py_vars.get(fid, {}) or self._py_bound(fid, call.src_id, r)):
|
|
2139
|
+
tid = self._py_var(fid, r) # a module-level variable, of this file or imported
|
|
2140
|
+
if tid:
|
|
2141
|
+
return tid, True
|
|
2142
|
+
# A bare identifier: a field of the enclosing type chain, or a type name (static call).
|
|
2143
|
+
for t in self._chain(call.enclosing_type):
|
|
2144
|
+
tn = self.field_type.get(t, {}).get(r)
|
|
2145
|
+
if tn:
|
|
2146
|
+
return self._type(lang, tn, call.src_id), True
|
|
2147
|
+
tid = self._type(lang, r, call.src_id)
|
|
2148
|
+
if tid:
|
|
2149
|
+
return tid, True
|
|
2150
|
+
if lang == "csharp" and r[:1].isupper() and not any(
|
|
2151
|
+
r in self.field_names.get(t, ()) for t in self._chain(call.enclosing_type)):
|
|
2152
|
+
return None, True # PascalCase, not a member, not ours: a type from outside (File.Open)
|
|
2153
|
+
return None, False
|
|
2154
|
+
|
|
2155
|
+
WRAPPERS = {"Task", "ValueTask", "Task`1", "ValueTask`1", "Nullable`1"}
|
|
2156
|
+
|
|
2157
|
+
def _returned_type(self, lang: str, inner: CallSite, depth: int = 0) -> Optional[tuple]:
|
|
2158
|
+
"""The type of a call's result, from the declared return type of what it resolves to.
|
|
2159
|
+
Returns (type id or None, known) like _receiver_type, or None when nothing can be said."""
|
|
2160
|
+
key = self._ckey(inner)
|
|
2161
|
+
if key in self._chain_memo:
|
|
2162
|
+
return self._chain_memo[key]
|
|
2163
|
+
self._chain_memo[key] = None
|
|
2164
|
+
if depth > 6:
|
|
2165
|
+
return None
|
|
2166
|
+
if inner.attr:
|
|
2167
|
+
out = self._attr_type(lang, inner)
|
|
2168
|
+
self._chain_memo[key] = out
|
|
2169
|
+
return out
|
|
2170
|
+
saved = (getattr(self, "_guessed", False), getattr(self, "_call", None), getattr(self, "_last_src", None))
|
|
2171
|
+
self._guessed = False
|
|
2172
|
+
stats_before = {k: dict(v) for k, v in self.stats.items()}
|
|
2173
|
+
self._call = inner
|
|
2174
|
+
fid = self.file_of.get(inner.src_id)
|
|
2175
|
+
targets = self._resolve_call(lang, fid, inner) if fid else None
|
|
2176
|
+
guessed = self._guessed
|
|
2177
|
+
self._guessed, self._call, self._last_src = saved[0], saved[1], saved[2]
|
|
2178
|
+
for k, v in stats_before.items(): # the inner call is counted when its own turn comes
|
|
2179
|
+
self.stats[k].clear()
|
|
2180
|
+
self.stats[k].update(v)
|
|
2181
|
+
out = None
|
|
2182
|
+
if targets and not guessed:
|
|
2183
|
+
t = targets[0]
|
|
2184
|
+
if t.name in CTORS:
|
|
2185
|
+
out = (t.parent_id, True)
|
|
2186
|
+
elif lang == "python" and t.attrs.get("returns_fn") and f"{t.id}.{t.attrs['returns_fn']}" in self.nodes:
|
|
2187
|
+
out = (f"{t.id}.{t.attrs['returns_fn']}", True) # a function value: calling it runs that function
|
|
2188
|
+
elif lang in FILE_MODULE:
|
|
2189
|
+
if t.attrs.get("returns"):
|
|
2190
|
+
tid = self._type(lang, t.attrs["returns"], t.id)
|
|
2191
|
+
out = (tid, True) if tid else None
|
|
2192
|
+
else:
|
|
2193
|
+
names = [n for n in t.attrs.get("returns_names") or [] if n not in self.WRAPPERS]
|
|
2194
|
+
if names and names[0] not in (t.attrs.get("type_params") or []):
|
|
2195
|
+
tid = self._type(lang, names[0], t.id)
|
|
2196
|
+
owner_params = re.findall(r"[A-Za-z_]\w*", (self.nodes[t.parent_id].attrs.get("signature") or "").split(":")[0].partition("<")[2]) \
|
|
2197
|
+
if t.parent_id in self.nodes else []
|
|
2198
|
+
if tid:
|
|
2199
|
+
out = (tid, True)
|
|
2200
|
+
elif names[0].split("`")[0] not in owner_params and names[0] not in ("void", "var", "dynamic", "object"):
|
|
2201
|
+
out = (None, True) # a declared type from outside the workspace
|
|
2202
|
+
self._chain_memo[key] = out
|
|
2203
|
+
return out
|
|
2204
|
+
|
|
2205
|
+
def _extensions(self, lang: str, fid: str, tid: str, name: str, argc: int) -> list[Node]:
|
|
2206
|
+
"""Extension methods called on a receiver of a known type: static methods whose `this` parameter is that type."""
|
|
2207
|
+
cands = [c for c in self.by_name.get((lang, name), ()) if c.attrs.get("is_extension") and self._can_see(fid, c.id)]
|
|
2208
|
+
if not cands:
|
|
2209
|
+
return []
|
|
2210
|
+
# Foo and Foo<T> are different receivers: compare the name together with its generic arity.
|
|
2211
|
+
names = {self.nodes[t].name.split("`")[0] + (f"`{_arity(t)}" if _arity(t) else "")
|
|
2212
|
+
for t in self._chain(tid) if t in self.nodes}
|
|
2213
|
+
exact = [c for c in cands if (c.attrs.get("param_types") or [""])[0] in names]
|
|
2214
|
+
loose = [c for c in cands if (c.attrs.get("param_types") or [""])[0] in (c.attrs.get("type_params") or [])]
|
|
2215
|
+
pool = exact or loose
|
|
2216
|
+
if not pool:
|
|
2217
|
+
return []
|
|
2218
|
+
if not exact:
|
|
2219
|
+
self._guessed = True # `this TBuilder builder`: the constraint is not checked
|
|
2220
|
+
return self._pick(pool, argc, skip=1)
|
|
2221
|
+
|
|
2222
|
+
def _resolve_calls(self) -> None:
|
|
2223
|
+
self._chain_memo: dict = {}
|
|
2224
|
+
self._call = None
|
|
2225
|
+
self._arg_log: list = [] # Python: (function, a call that reaches it), for _py_argument_flow
|
|
2226
|
+
self._pending: list = [] # Python: calls nothing resolved, for _py_argument_flow
|
|
2227
|
+
for (tid, fname), method in self.field_calls.items():
|
|
2228
|
+
# self.config = self.make_config(): the field holds what that method is declared to return.
|
|
2229
|
+
if fname in self.field_type.get(tid, {}):
|
|
2230
|
+
continue
|
|
2231
|
+
for m in self._methods(tid, method, -1):
|
|
2232
|
+
if m.attrs.get("returns"):
|
|
2233
|
+
self.field_type[tid][fname] = m.attrs["returns"]
|
|
2234
|
+
self.field_types_global[(m.language, fname)].add(m.attrs["returns"])
|
|
2235
|
+
break
|
|
2236
|
+
# Names seen on receivers of a known outside type (List.Add, dict.get). A call to such a
|
|
2237
|
+
# name on a receiver of unknown type is never guessed.
|
|
2238
|
+
self.outside_names: set = set()
|
|
2239
|
+
for a in BY_LANGUAGE.values():
|
|
2240
|
+
self.outside_names.update((a.LANGUAGE, n) for n in getattr(a, "COMMON_METHODS", ()))
|
|
2241
|
+
self._py_fixtures()
|
|
2242
|
+
for fid, res in self._calls_pass(1):
|
|
2243
|
+
lang = self.file_lang[fid]
|
|
2244
|
+
for call in self._open(fid, res).calls:
|
|
2245
|
+
if call.receiver not in (None, "this", "base") and call.name != ".ctor":
|
|
2246
|
+
if lang in FILE_MODULE and (self._py_bound(fid, call.src_id, call.receiver) if lang == "python" else call.receiver in self.py_names[fid]):
|
|
2247
|
+
continue
|
|
2248
|
+
if call.ref:
|
|
2249
|
+
continue
|
|
2250
|
+
tid, known = self._receiver_type(lang, call)
|
|
2251
|
+
if known and tid is None:
|
|
2252
|
+
self.outside_names.add((lang, call.name))
|
|
2253
|
+
for fid, res in self._calls_pass(2):
|
|
2254
|
+
lang = self.file_lang[fid]
|
|
2255
|
+
adapter = BY_LANGUAGE[lang].NAME
|
|
2256
|
+
st = self.stats[adapter]
|
|
2257
|
+
for call in self._open(fid, res).calls:
|
|
2258
|
+
if call.ref:
|
|
2259
|
+
# A function handed over by name. Linked when the name is one of ours; otherwise it was a value.
|
|
2260
|
+
self._guessed, self._call = False, None
|
|
2261
|
+
before = dict(st)
|
|
2262
|
+
targets = [t for t in (self._resolve_call(lang, fid, call) or []) if t.kind == "callable"]
|
|
2263
|
+
st.clear()
|
|
2264
|
+
st.update(before)
|
|
2265
|
+
if targets and not self._guessed:
|
|
2266
|
+
st["references_linked"] += 1
|
|
2267
|
+
for t in targets:
|
|
2268
|
+
self.calls.append((call.src_id, t.id, "reference", "heuristic", call.line))
|
|
2269
|
+
self.call_col[(call.src_id, t.id, call.line)] = call.col
|
|
2270
|
+
continue
|
|
2271
|
+
st["calls_total"] += 1
|
|
2272
|
+
self._guessed = False
|
|
2273
|
+
self._call = call
|
|
2274
|
+
targets = self._resolve_call(lang, fid, call)
|
|
2275
|
+
if targets is None:
|
|
2276
|
+
st["calls_external"] += 1
|
|
2277
|
+
elif not targets:
|
|
2278
|
+
st["calls_unresolved"] += 1
|
|
2279
|
+
else:
|
|
2280
|
+
st["calls_resolved"] += 1
|
|
2281
|
+
for t in targets:
|
|
2282
|
+
dispatch = "virtual" if t.attrs.get("is_virtual") and lang == "csharp" else "static"
|
|
2283
|
+
self.calls.append((call.src_id, t.id, dispatch,
|
|
2284
|
+
"guess" if self._guessed else "heuristic", call.line))
|
|
2285
|
+
self.call_col[(call.src_id, t.id, call.line)] = call.col
|
|
2286
|
+
if lang == "python":
|
|
2287
|
+
if targets and not self._guessed:
|
|
2288
|
+
for t in targets:
|
|
2289
|
+
self._arg_log.append((t.id, call))
|
|
2290
|
+
elif not targets:
|
|
2291
|
+
self._pending.append(call)
|
|
2292
|
+
self._shut()
|
|
2293
|
+
# Kept as lists on the indexer, so an incremental run records each file's share with its turn and replays it.
|
|
2294
|
+
incoming: dict[str, list] = defaultdict(list) # Python: function -> the calls that reach it
|
|
2295
|
+
for tid, call in self._arg_log:
|
|
2296
|
+
incoming[tid].append(call)
|
|
2297
|
+
self._py_argument_flow(incoming, self._pending)
|
|
2298
|
+
|
|
2299
|
+
def _py_argument_flow(self, incoming: dict, pending: list) -> None:
|
|
2300
|
+
"""Calls made on a value handed in from outside: `app(environ, start)` on a parameter, `self.repo.save()` on
|
|
2301
|
+
a field set from one. The value's class is whatever the callers pass, followed back through parameters,
|
|
2302
|
+
fields set from parameters and `*args` passed on. Linked to that class's method (`__call__` when the value
|
|
2303
|
+
itself is called), as a guess, since a caller the map does not see may pass something else."""
|
|
2304
|
+
if not pending:
|
|
2305
|
+
return
|
|
2306
|
+
memo: dict = {}
|
|
2307
|
+
active: set = set()
|
|
2308
|
+
cut = [0, 0] # answers left incomplete by a cycle or a limit; questions asked for the current call
|
|
2309
|
+
|
|
2310
|
+
def param_owner(src: str, name: str) -> Optional[tuple]:
|
|
2311
|
+
cur = src
|
|
2312
|
+
while cur in self.nodes and self.nodes[cur].kind == "callable":
|
|
2313
|
+
params = self.nodes[cur].attrs.get("params") or []
|
|
2314
|
+
if name in params:
|
|
2315
|
+
return cur, params.index(name)
|
|
2316
|
+
cur = self.nodes[cur].parent_id
|
|
2317
|
+
return None
|
|
2318
|
+
|
|
2319
|
+
def narrowed(src: str, owner: str, name: str) -> bool:
|
|
2320
|
+
"""Is the parameter tested with isinstance() between the call and the function that takes it?"""
|
|
2321
|
+
cur = src
|
|
2322
|
+
while cur in self.nodes and self.nodes[cur].kind == "callable":
|
|
2323
|
+
if name in (self.nodes[cur].attrs.get("narrowed") or ()):
|
|
2324
|
+
return True
|
|
2325
|
+
if cur == owner:
|
|
2326
|
+
return False
|
|
2327
|
+
cur = self.nodes[cur].parent_id
|
|
2328
|
+
return False
|
|
2329
|
+
|
|
2330
|
+
def field_types(type_id: Optional[str], name: str, depth: int) -> set:
|
|
2331
|
+
for t in self._chain(type_id):
|
|
2332
|
+
f = self.nodes.get(f"{t}.{name}")
|
|
2333
|
+
if f is None or f.kind != "field":
|
|
2334
|
+
continue
|
|
2335
|
+
tn = self.field_type.get(t, {}).get(name)
|
|
2336
|
+
tid = self._type("python", tn, f.id) if tn else None
|
|
2337
|
+
if tid:
|
|
2338
|
+
return {tid}
|
|
2339
|
+
out = set()
|
|
2340
|
+
for fn, pname in f.attrs.get("from_params") or []: # self.app = app, in the constructor
|
|
2341
|
+
owner = param_owner(fn, pname)
|
|
2342
|
+
out |= arg_types(owner[0], owner[1], depth + 1) if owner else set()
|
|
2343
|
+
return out
|
|
2344
|
+
return set()
|
|
2345
|
+
|
|
2346
|
+
def arg_types(fn_id: str, pos: int, depth: int = 0) -> set:
|
|
2347
|
+
"""Classes of ours passed as the pos-th positional argument of fn_id."""
|
|
2348
|
+
key = (fn_id, pos)
|
|
2349
|
+
if key in memo:
|
|
2350
|
+
return memo[key]
|
|
2351
|
+
fn = self.nodes.get(fn_id)
|
|
2352
|
+
cut[1] += 1
|
|
2353
|
+
if fn is None or key in active or depth > 8 or cut[1] > 5000:
|
|
2354
|
+
cut[0] += 1 # not cached: the same question asked from higher up may get further
|
|
2355
|
+
return set()
|
|
2356
|
+
params, star = fn.attrs.get("params") or [], fn.attrs.get("star_at")
|
|
2357
|
+
if pos < len(params) and (star is None or pos < star):
|
|
2358
|
+
declared = (fn.attrs.get("ptypes") or {}).get(params[pos])
|
|
2359
|
+
tid = self._type("python", declared, fn_id) if declared else None
|
|
2360
|
+
if tid:
|
|
2361
|
+
memo[key] = {tid}
|
|
2362
|
+
return memo[key]
|
|
2363
|
+
active.add(key)
|
|
2364
|
+
before, out = cut[0], set()
|
|
2365
|
+
for c in incoming.get(fn_id, ()):
|
|
2366
|
+
hints = c.args or ()
|
|
2367
|
+
spread = next((k for k, h in enumerate(hints[:pos + 1]) if h == "@*"), None)
|
|
2368
|
+
if spread is not None:
|
|
2369
|
+
caller = self.nodes.get(c.src_id)
|
|
2370
|
+
if caller is not None and caller.attrs.get("star_at") is not None:
|
|
2371
|
+
out |= arg_types(c.src_id, caller.attrs["star_at"] + pos - spread, depth + 1)
|
|
2372
|
+
continue
|
|
2373
|
+
h = hints[pos] if pos < len(hints) else None
|
|
2374
|
+
if h is None:
|
|
2375
|
+
continue
|
|
2376
|
+
if h == "@self":
|
|
2377
|
+
out |= {c.enclosing_type} if c.enclosing_type else set()
|
|
2378
|
+
elif h.startswith("@p:"):
|
|
2379
|
+
owner = param_owner(c.src_id, h[3:])
|
|
2380
|
+
out |= arg_types(owner[0], owner[1], depth + 1) if owner else set()
|
|
2381
|
+
elif h.startswith("@f:"):
|
|
2382
|
+
out |= field_types(c.enclosing_type, h[3:], depth)
|
|
2383
|
+
else:
|
|
2384
|
+
tid = self._type("python", h, c.src_id)
|
|
2385
|
+
out |= {tid} if tid else set()
|
|
2386
|
+
active.discard(key)
|
|
2387
|
+
if cut[0] == before:
|
|
2388
|
+
memo[key] = out
|
|
2389
|
+
return out
|
|
2390
|
+
|
|
2391
|
+
st = self.stats[BY_LANGUAGE["python"].NAME]
|
|
2392
|
+
for call in pending:
|
|
2393
|
+
r, method = call.receiver, call.name
|
|
2394
|
+
types: set = set()
|
|
2395
|
+
cut[1] = 0
|
|
2396
|
+
if r is None or r not in ("this", "base", "?"):
|
|
2397
|
+
owner = param_owner(call.src_id, r or call.name)
|
|
2398
|
+
if owner is not None and narrowed(call.src_id, owner[0], r or call.name):
|
|
2399
|
+
owner = None
|
|
2400
|
+
if owner is not None and (r is None or call.receiver_type is None or not self._type("python", call.receiver_type, call.src_id)):
|
|
2401
|
+
types = arg_types(*owner)
|
|
2402
|
+
method = "__call__" if r is None else method
|
|
2403
|
+
elif r == "this" and call.enclosing_type and call.name in self.field_names.get(call.enclosing_type, ()):
|
|
2404
|
+
types, method = field_types(call.enclosing_type, call.name, 0), "__call__" # self.handler(x)
|
|
2405
|
+
if r is not None and r.startswith(".") and call.enclosing_type:
|
|
2406
|
+
types = field_types(call.enclosing_type, r[1:], 0)
|
|
2407
|
+
for tid in sorted(types):
|
|
2408
|
+
self._call = call
|
|
2409
|
+
for t in self._methods(tid, method, call.argc):
|
|
2410
|
+
st["calls_by_argument_flow"] += 1
|
|
2411
|
+
self.calls.append((call.src_id, t.id, "static", "guess", call.line))
|
|
2412
|
+
self.call_col[(call.src_id, t.id, call.line)] = call.col
|
|
2413
|
+
|
|
2414
|
+
def _calls_pass(self, phase: int):
|
|
2415
|
+
"""The files whose calls _resolve_calls goes through, in its first pass (names seen on outside types) and
|
|
2416
|
+
its second (the calls themselves). Every file, in order; leyline.incremental narrows it to the files an
|
|
2417
|
+
edit can have changed and fills in the rest from the last run."""
|
|
2418
|
+
return self.results.items()
|
|
2419
|
+
|
|
2420
|
+
_DECL = [ # (pattern, group of the name, group of the type): how typed languages write a variable's type
|
|
2421
|
+
(re.compile(r"\b([A-Z]\w*)(?:<[^<>;=()]*>)?(?:\[\])?\??\s*[*&]?\s+(\w+)\s*(?=[=;,)]|$)"), 2, 1), # Foo x (C#, Java, C++)
|
|
2422
|
+
(re.compile(r"\b(\w+)\s*:\s*&?(?:mut\s+|readonly\s+)?\(?(?:[a-z_]\w*(?:::|\.))*([A-Z]\w*)"), 1, 2), # x: Foo, x: &mod::Foo (TS, Kotlin, Swift, Rust, Python)
|
|
2423
|
+
(re.compile(r"\b(\w+)\s*(?::=|=)\s*(?:new\s+|&|await\s+)?([A-Z]\w*)(?:<[^<>]*>)?(?:::new)?\s*[({]"), 1, 2), # x = new Foo( / Foo{ / Foo::new(
|
|
2424
|
+
(re.compile(r"\b(\w+)\s+\*?(?:[a-z_]\w*\.)?([A-Z]\w*)\s*[,)]"), 1, 2), # (x *Foo, s *store.Store) (Go parameters)
|
|
2425
|
+
(re.compile(r"\bvar\s+(\w+)\s+\*?(?:[a-z_]\w*\.)?([A-Z]\w*)"), 1, 2), # var x Foo, var c store.Cache (Go)
|
|
2426
|
+
# built-in types, which tell an overload apart: int x, boolean b[], x int, err error
|
|
2427
|
+
(re.compile(r"\b(byte|short|int|long|float|double|boolean|bool|char|string|str|rune)(?:\[\])?\s+(\w+)\s*(?=[=;,)])"), 2, 1),
|
|
2428
|
+
(re.compile(r"\b(\w+)\s+(byte|int|int32|int64|uint|float32|float64|bool|string|rune|error)\s*[,)]"), 1, 2),
|
|
2429
|
+
]
|
|
2430
|
+
|
|
2431
|
+
def _generic_var_type(self, lang: str, src_id: str, var: str, enclosing_type: Optional[str]) -> Optional[str]:
|
|
2432
|
+
"""The type a variable or field is declared with, read from the text of the function and of its type."""
|
|
2433
|
+
if not re.fullmatch(r"[A-Za-z_]\w*", var):
|
|
2434
|
+
return None
|
|
2435
|
+
for holder in (src_id, enclosing_type):
|
|
2436
|
+
if not holder or holder not in self.nodes:
|
|
2437
|
+
continue
|
|
2438
|
+
decls = self._generic_decls(lang, holder)[0]
|
|
2439
|
+
if var in decls:
|
|
2440
|
+
if decls[var] is None:
|
|
2441
|
+
return "external" # declared with a type that is not in the repo
|
|
2442
|
+
return self._type(lang, decls[var], src_id) or (self.types_by_name.get((lang, decls[var])) or [None])[0]
|
|
2443
|
+
return None
|
|
2444
|
+
|
|
2445
|
+
def _generic_decls(self, lang: str, holder: str) -> tuple:
|
|
2446
|
+
"""({variable: repo type name, or None for a type from outside}, {variable: type name as written})."""
|
|
2447
|
+
got = self._decl_cache.get(holder)
|
|
2448
|
+
if got is None:
|
|
2449
|
+
n = self.nodes[holder]
|
|
2450
|
+
fid = self.file_of.get(holder)
|
|
2451
|
+
read = self._decls_read(fid).get(holder)
|
|
2452
|
+
if read is not None and read[:2] == (n.span_start, n.span_end):
|
|
2453
|
+
found = read[2] # read by the parse worker from the same text
|
|
2454
|
+
else:
|
|
2455
|
+
found = _declared(self._file_text(fid) if fid else [], n.span_start, n.span_end)
|
|
2456
|
+
decls, raw = {}, {}
|
|
2457
|
+
for name, tnames in found.items():
|
|
2458
|
+
raw[name] = tnames[0]
|
|
2459
|
+
typed = [t for t in tnames if t not in NOT_TYPES]
|
|
2460
|
+
if typed:
|
|
2461
|
+
# the first type written for it, or, if that one is not in the repo, the first after it that is
|
|
2462
|
+
decls[name] = next((t for t in typed if self.types_by_name.get((lang, t))), None)
|
|
2463
|
+
if len(self._decl_cache) >= 4096:
|
|
2464
|
+
# Asked about the functions of the file being resolved; made again the same when asked later.
|
|
2465
|
+
# Kept for every function, it was hundreds of MB on a large repository.
|
|
2466
|
+
self._decl_cache.clear()
|
|
2467
|
+
got = self._decl_cache[holder] = (decls, raw)
|
|
2468
|
+
return got
|
|
2469
|
+
|
|
2470
|
+
_INTS = re.compile(r"(?i)^(int|long|short|byte|integer|bigint|biginteger|[iu](8|16|32|64|128|size)|u?int(8|16|32|64)?|rune)$")
|
|
2471
|
+
_FLOATS = re.compile(r"(?i)^(float|double|f32|f64|float32|float64|decimal|bigdecimal|number)$")
|
|
2472
|
+
_STRS = re.compile(r"(?i)^(string|str|charsequence)$")
|
|
2473
|
+
_BOOLS = re.compile(r"(?i)^(bool|boolean)$")
|
|
2474
|
+
_PRIMS = {"int", "long", "short", "byte", "float", "double", "boolean", "char", "bool"}
|
|
2475
|
+
_ANY = {"Object", "object", "any", "Any", "AnyObject", "dynamic", "interface", "unknown"}
|
|
2476
|
+
|
|
2477
|
+
def _supers(self, tid: str) -> set:
|
|
2478
|
+
out, queue = set(), [tid]
|
|
2479
|
+
while queue:
|
|
2480
|
+
t = queue.pop()
|
|
2481
|
+
if t not in out:
|
|
2482
|
+
out.add(t)
|
|
2483
|
+
queue.extend(self.bases.get(t, []))
|
|
2484
|
+
return out
|
|
2485
|
+
|
|
2486
|
+
def _arg_fits(self, lang: str, hint: str, ptype: str) -> int:
|
|
2487
|
+
"""How well an argument fits a declared parameter type: 2 the same, 1 compatible, 0 cannot tell, -1 cannot be."""
|
|
2488
|
+
p = ptype.rstrip(".")
|
|
2489
|
+
if not p or not hint or hint == "fn":
|
|
2490
|
+
return 0
|
|
2491
|
+
repo_p = self.types_by_name.get((lang, p))
|
|
2492
|
+
kinds = {"string": self._STRS, "int": self._INTS, "float": self._FLOATS, "bool": self._BOOLS}
|
|
2493
|
+
boxed, named = False, hint
|
|
2494
|
+
if hint not in kinds and hint not in ("char", "null") and not self.types_by_name.get((lang, hint)):
|
|
2495
|
+
# a built-in type by name (int, Integer, String): the same as the literal of that kind
|
|
2496
|
+
kind = "char" if hint.lower() in ("char", "character") else \
|
|
2497
|
+
next((k for k, pat in kinds.items() if pat.match(hint) and hint.lower() not in ("number", "decimal")), None)
|
|
2498
|
+
if kind:
|
|
2499
|
+
boxed, hint = hint[:1].isupper() and kind != "string", kind
|
|
2500
|
+
if hint in kinds or hint == "char":
|
|
2501
|
+
# a literal is the built-in type exactly (int, bool); its boxed or wider form takes it too
|
|
2502
|
+
exact = 2 if p[:1].islower() != boxed or hint == "string" else 1
|
|
2503
|
+
if hint == "char" and p.lower() in ("char", "character", "rune"):
|
|
2504
|
+
return exact
|
|
2505
|
+
if hint in kinds and kinds[hint].match(p):
|
|
2506
|
+
if named.lower() == p.lower() and named != hint:
|
|
2507
|
+
return 3 # the very type named: Float.NaN to float rather than double
|
|
2508
|
+
if named == "float" and re.fullmatch(r"(?i)float|f32|float32", p):
|
|
2509
|
+
return 1 # a decimal literal is a double unless marked otherwise
|
|
2510
|
+
return exact if p.lower() not in ("number", "decimal", "bigdecimal", "biginteger", "bigint") else 1
|
|
2511
|
+
if hint in ("int", "char") and self._FLOATS.match(p) or hint == "char" and self._INTS.match(p):
|
|
2512
|
+
return 1 # widened
|
|
2513
|
+
other = any(k.match(p) for k in kinds.values()) or p in self._PRIMS
|
|
2514
|
+
return -1 if other or repo_p else 0
|
|
2515
|
+
if hint == "null":
|
|
2516
|
+
return -1 if p in self._PRIMS else 0
|
|
2517
|
+
if any(k.match(p) for k in kinds.values()) or p in self._PRIMS:
|
|
2518
|
+
return -1 # a type that is not a built-in one passed where a built-in one is taken
|
|
2519
|
+
if hint == p:
|
|
2520
|
+
return 2
|
|
2521
|
+
repo_h = self.types_by_name.get((lang, hint))
|
|
2522
|
+
if repo_h and repo_p:
|
|
2523
|
+
return 1 if any(t in self._supers(h) for h in repo_h for t in repo_p) else -1
|
|
2524
|
+
if repo_h and (p in self._PRIMS or any(k.match(p) for k in kinds.values())):
|
|
2525
|
+
return -1
|
|
2526
|
+
if repo_p and not repo_h and (len(hint) > 2 or not hint.isupper()):
|
|
2527
|
+
return -1 # a type from outside cannot be one of ours (a one-letter name is a type parameter)
|
|
2528
|
+
if repo_h and not repo_p and p not in self._ANY and (len(p) > 2 or not p.isupper()):
|
|
2529
|
+
# one of ours passed where a type from outside is taken: only if it says it extends that type
|
|
2530
|
+
said = " ".join(self.nodes[t].attrs.get("signature") or "" for h in repo_h for t in self._supers(h) if t in self.nodes)
|
|
2531
|
+
return 1 if re.search(rf"\b{re.escape(p)}\b", said) else -1
|
|
2532
|
+
return 0
|
|
2533
|
+
|
|
2534
|
+
def _generic_returns(self, lang: str, inner: CallSite) -> Optional[tuple]:
|
|
2535
|
+
"""_returned_type for the generic resolver, without the edges resolving the inner call would record twice."""
|
|
2536
|
+
if inner.receiver is None and self.types_by_name.get((lang, inner.name)):
|
|
2537
|
+
return (self.types_by_name[(lang, inner.name)][0], True) # Foo(): a constructor called by the type's name
|
|
2538
|
+
n_edges = len(self.edges)
|
|
2539
|
+
out = self._returned_type(lang, inner)
|
|
2540
|
+
del self.edges[n_edges:]
|
|
2541
|
+
return out
|
|
2542
|
+
|
|
2543
|
+
def _generic_return_name(self, lang: str, inner: CallSite) -> Optional[str]:
|
|
2544
|
+
"""The type name a call's target declares it returns, from outside the repo or not (None when the call
|
|
2545
|
+
does not resolve, or when what it returns is a type parameter)."""
|
|
2546
|
+
key = ("ret", self._ckey(inner))
|
|
2547
|
+
if key in self._chain_memo:
|
|
2548
|
+
return self._chain_memo[key]
|
|
2549
|
+
self._chain_memo[key] = None
|
|
2550
|
+
if inner.receiver is None and self.types_by_name.get((lang, inner.name)):
|
|
2551
|
+
out = inner.name
|
|
2552
|
+
else:
|
|
2553
|
+
saved = (getattr(self, "_guessed", False), getattr(self, "_call", None))
|
|
2554
|
+
stats_before = {k: dict(v) for k, v in self.stats.items()}
|
|
2555
|
+
n_edges = len(self.edges)
|
|
2556
|
+
self._guessed = False
|
|
2557
|
+
fid = self.file_of.get(inner.src_id)
|
|
2558
|
+
targets = self._resolve_call(lang, fid, inner) if fid else None
|
|
2559
|
+
guessed = self._guessed
|
|
2560
|
+
self._guessed, self._call = saved
|
|
2561
|
+
for k, v in stats_before.items():
|
|
2562
|
+
self.stats[k].clear()
|
|
2563
|
+
self.stats[k].update(v)
|
|
2564
|
+
del self.edges[n_edges:]
|
|
2565
|
+
names = {(t.attrs.get("returns_names") or [None])[0] for t in targets or ()
|
|
2566
|
+
if (t.attrs.get("returns_names") or [None])[0] not in (t.attrs.get("type_params") or ())}
|
|
2567
|
+
out = names.pop() if targets and not guessed and len(names) == 1 else None
|
|
2568
|
+
self._chain_memo[key] = out
|
|
2569
|
+
return out
|
|
2570
|
+
|
|
2571
|
+
def _hint_type(self, lang: str, call: CallSite, h) -> Optional[str]:
|
|
2572
|
+
"""The type name an argument hint stands for: `$x` a variable, `$a.b` a field, a call its return type."""
|
|
2573
|
+
if isinstance(h, CallSite):
|
|
2574
|
+
return self._generic_return_name(lang, h)
|
|
2575
|
+
if not isinstance(h, str) or not h.startswith("$"):
|
|
2576
|
+
return h if isinstance(h, str) else None
|
|
2577
|
+
var, _, field = h[1:].partition(".")
|
|
2578
|
+
if field:
|
|
2579
|
+
if not self.types_by_name.get((lang, var)) and any(k.match(var) for k in (self._INTS, self._FLOATS, self._BOOLS)):
|
|
2580
|
+
return var # Integer.MAX_VALUE, Double.NaN: a constant of that built-in type
|
|
2581
|
+
if var in ("this", "self"):
|
|
2582
|
+
tids = [call.enclosing_type] if call.enclosing_type else []
|
|
2583
|
+
else:
|
|
2584
|
+
tids = self.types_by_name.get((lang, var)) or []
|
|
2585
|
+
if not tids:
|
|
2586
|
+
tid = self._generic_var_type(lang, call.src_id, var, call.enclosing_type)
|
|
2587
|
+
tids = [tid] if tid and tid != "external" else []
|
|
2588
|
+
for tid in tids:
|
|
2589
|
+
for t in self._chain(tid):
|
|
2590
|
+
if field in self.field_type.get(t, {}):
|
|
2591
|
+
return self.field_type[t][field]
|
|
2592
|
+
return None
|
|
2593
|
+
for holder in (call.src_id, call.enclosing_type):
|
|
2594
|
+
if holder and holder in self.nodes:
|
|
2595
|
+
raw = self._generic_decls(lang, holder)[1]
|
|
2596
|
+
if var in raw:
|
|
2597
|
+
return raw[var]
|
|
2598
|
+
return None
|
|
2599
|
+
|
|
2600
|
+
def _by_arg_types(self, lang: str, call: CallSite, fns: list) -> list:
|
|
2601
|
+
"""Overloads with as many parameters: keep those whose parameter types fit what the arguments show."""
|
|
2602
|
+
hints = [self._hint_type(lang, call, h) for h in call.args]
|
|
2603
|
+
if not any(hints):
|
|
2604
|
+
return fns
|
|
2605
|
+
scored = []
|
|
2606
|
+
for f in fns:
|
|
2607
|
+
types, score = f.attrs.get("param_types") or [], 0
|
|
2608
|
+
for i, h in enumerate(hints):
|
|
2609
|
+
p = types[i] if i < len(types) else (types[-1] if types and types[-1].endswith("...") else "")
|
|
2610
|
+
v = self._arg_fits(lang, h, p) if h else 0
|
|
2611
|
+
if v < 0:
|
|
2612
|
+
score = -1
|
|
2613
|
+
break
|
|
2614
|
+
score += v
|
|
2615
|
+
scored.append((score, f))
|
|
2616
|
+
best = max(sc for sc, _ in scored)
|
|
2617
|
+
return [f for sc, f in scored if sc == best] if best >= 0 else fns
|
|
2618
|
+
|
|
2619
|
+
def _file_text(self, fid: str) -> list[str]:
|
|
2620
|
+
if fid not in self._text_cache:
|
|
2621
|
+
if len(self._text_cache) >= 8:
|
|
2622
|
+
# Calls are resolved a file at a time, so a few files are enough; keeping every file's text
|
|
2623
|
+
# cost hundreds of MB on a large repo.
|
|
2624
|
+
self._text_cache.clear()
|
|
2625
|
+
try:
|
|
2626
|
+
self._text_cache[fid] = source_lines(self.repos[_repo_of(fid)] / self.nodes[fid].path)
|
|
2627
|
+
except OSError:
|
|
2628
|
+
self._text_cache[fid] = []
|
|
2629
|
+
return self._text_cache[fid]
|
|
2630
|
+
|
|
2631
|
+
def _near(self, fid: str) -> set:
|
|
2632
|
+
"""Files this one imports, and the files those import in turn (a package's index re-exporting its parts)."""
|
|
2633
|
+
key = ("near", fid)
|
|
2634
|
+
got = self._file_set(key)
|
|
2635
|
+
if got is None:
|
|
2636
|
+
first = set(self.import_targets.get(fid, ()))
|
|
2637
|
+
got = self._keep_set(key, first | {t for f in first for t in self.import_targets.get(f, ())})
|
|
2638
|
+
return got
|
|
2639
|
+
|
|
2640
|
+
def _name_index(self, lang: str, name: str) -> dict:
|
|
2641
|
+
"""Where the callables of one name are declared, looked up by file and directory. A common name (get, run)
|
|
2642
|
+
has thousands of declarations in a large repo, and scanning them for every call was quadratic."""
|
|
2643
|
+
key = (lang, name)
|
|
2644
|
+
ix = self._name_ix.get(key)
|
|
2645
|
+
if ix is None:
|
|
2646
|
+
ix = {"free": [], "typed": [], "free_file": defaultdict(list), "typed_file": defaultdict(list),
|
|
2647
|
+
"free_dir": {}, "free_dir_base": {}, "stem": {}, "stem_free": {}, "owners": set(), "typed_owners": set()}
|
|
2648
|
+
for i, c in enumerate(self.by_name.get(key, ())):
|
|
2649
|
+
f = self.file_of.get(c.id)
|
|
2650
|
+
typed = bool(c.attrs.get("type_id"))
|
|
2651
|
+
ix["owners"].add(c.parent_id)
|
|
2652
|
+
if typed:
|
|
2653
|
+
ix["typed"].append(c)
|
|
2654
|
+
ix["typed_file"][f].append((i, c))
|
|
2655
|
+
ix["typed_owners"].add(c.parent_id)
|
|
2656
|
+
else:
|
|
2657
|
+
ix["free"].append(c)
|
|
2658
|
+
ix["free_file"][f].append((i, c))
|
|
2659
|
+
if f is None:
|
|
2660
|
+
continue
|
|
2661
|
+
d = self.nodes[f].path.rpartition("/")[0]
|
|
2662
|
+
if not typed:
|
|
2663
|
+
ix["free_dir"].setdefault(d, c)
|
|
2664
|
+
ix["free_dir_base"].setdefault(d.rsplit("/", 1)[-1], c)
|
|
2665
|
+
ix["stem"].setdefault(self.nodes[f].name.rsplit(".", 1)[0], c)
|
|
2666
|
+
if not typed:
|
|
2667
|
+
ix["stem_free"].setdefault(self.nodes[f].name.rsplit(".", 1)[0], c)
|
|
2668
|
+
self._name_ix[key] = ix
|
|
2669
|
+
return ix
|
|
2670
|
+
|
|
2671
|
+
@staticmethod
|
|
2672
|
+
def _in_files(by_file: dict, files, also: Optional[str] = None) -> list:
|
|
2673
|
+
"""The declarations in any of these files (or in `also`), in declaration order. Walks whichever side is
|
|
2674
|
+
smaller: a Go package's files all import each other, so `files` can be thousands long for every call."""
|
|
2675
|
+
if len(by_file) < len(files):
|
|
2676
|
+
hits = [ic for f, ics in by_file.items() if f in files or f == also for ic in ics]
|
|
2677
|
+
else:
|
|
2678
|
+
hits = [ic for f in files for ic in by_file.get(f, ())]
|
|
2679
|
+
if also is not None and also not in files:
|
|
2680
|
+
hits.extend(by_file.get(also, ()))
|
|
2681
|
+
hits.sort(key=lambda ic: ic[0])
|
|
2682
|
+
return [c for _, c in hits]
|
|
2683
|
+
|
|
2684
|
+
def _generic_call(self, lang: str, fid: str, call: CallSite) -> Optional[list[Node]]:
|
|
2685
|
+
"""A call read by the generic adapter: only the name and what the text shows of the receiver are known.
|
|
2686
|
+
Nearest first: the caller's own type, its file, its directory, a type or package named by the receiver,
|
|
2687
|
+
then a name declared once in the whole repo, which is a guess."""
|
|
2688
|
+
name = call.name
|
|
2689
|
+
cands = self.by_name.get((lang, name), [])
|
|
2690
|
+
types = self.types_by_name.get((lang, name), [])
|
|
2691
|
+
if len(self.repos) > 1: # another repository's names are seen only through what this file imports
|
|
2692
|
+
reach = self._reach(fid)
|
|
2693
|
+
mine = lambda i: _repo_of(i) == _repo_of(fid) or self.file_of.get(i) in reach
|
|
2694
|
+
cands, types = [c for c in cands if mine(c.id)], [t for t in types if mine(t)]
|
|
2695
|
+
if not cands and not types:
|
|
2696
|
+
return None
|
|
2697
|
+
ix = self._name_index(lang, name)
|
|
2698
|
+
r = call.receiver
|
|
2699
|
+
|
|
2700
|
+
def by_args(fns):
|
|
2701
|
+
"""Overloads: keep those declared with as many parameters as the call passes, when that narrows it."""
|
|
2702
|
+
fns = list(fns)
|
|
2703
|
+
if len(fns) > 1:
|
|
2704
|
+
fns = [f for f in fns if not f.attrs.get("via_base")] or fns
|
|
2705
|
+
if len(fns) > 1 and call.argc >= 0:
|
|
2706
|
+
# A declaration with fewer parameters than the call passes cannot take it; of the rest, the one
|
|
2707
|
+
# with the fewest is likeliest (the others' extra parameters would need defaults).
|
|
2708
|
+
fit = [f for f in fns if f.attrs.get("params_seen") is None or f.attrs["params_seen"] >= call.argc]
|
|
2709
|
+
if fit and all(f.attrs.get("params_seen") is not None for f in fit):
|
|
2710
|
+
least = min(f.attrs["params_seen"] for f in fit)
|
|
2711
|
+
fit = [f for f in fit if f.attrs["params_seen"] == least]
|
|
2712
|
+
if len(fit) > 1 and call.args:
|
|
2713
|
+
fit = self._by_arg_types(lang, call, fit)
|
|
2714
|
+
return fit or fns
|
|
2715
|
+
return fns
|
|
2716
|
+
|
|
2717
|
+
def ctor_of(tids):
|
|
2718
|
+
out = []
|
|
2719
|
+
for t in tids:
|
|
2720
|
+
out += self.members.get(t, {}).get(name, []) or [m for c in CTORS for m in self.members.get(t, {}).get(c, [])]
|
|
2721
|
+
self.edges.append(Edge("instantiates", call.src_id, t, "heuristic"))
|
|
2722
|
+
return by_args(out) or None
|
|
2723
|
+
near_files = self._near(fid)
|
|
2724
|
+
here = self.nodes[fid].path.rpartition("/")[0]
|
|
2725
|
+
if r in (None, "this", "base"):
|
|
2726
|
+
chain = self._chain(call.enclosing_type)
|
|
2727
|
+
caller = self.nodes.get(call.src_id)
|
|
2728
|
+
if r is None and caller is not None and caller.attrs.get("explicit_self"):
|
|
2729
|
+
chain = [] # Go, Rust, Python: a member is reached only through the receiver
|
|
2730
|
+
for t in chain[1:] if r == "base" else chain:
|
|
2731
|
+
found = self.members.get(t, {}).get(name)
|
|
2732
|
+
if found:
|
|
2733
|
+
return by_args(found)
|
|
2734
|
+
if r is not None:
|
|
2735
|
+
return [] if cands else None
|
|
2736
|
+
local = ix["free_file"].get(fid) # declared in this file, then in this directory
|
|
2737
|
+
if local:
|
|
2738
|
+
return [local[0][1]]
|
|
2739
|
+
if here in ix["free_dir"]:
|
|
2740
|
+
return [ix["free_dir"][here]]
|
|
2741
|
+
if types and r is None:
|
|
2742
|
+
local_t = [t for t in types if self.file_of.get(t) == fid] or \
|
|
2743
|
+
[t for t in types if self.file_of.get(t) in near_files] or \
|
|
2744
|
+
[t for t in types if self.nodes[self.file_of.get(t, fid)].path.rpartition("/")[0] == here] or types
|
|
2745
|
+
if len(local_t) == 1:
|
|
2746
|
+
return ctor_of(local_t)
|
|
2747
|
+
free = ix["free"]
|
|
2748
|
+
imported = self._in_files(ix["free_file"], self.import_targets.get(fid, ())) or \
|
|
2749
|
+
self._in_files(ix["free_file"], near_files)
|
|
2750
|
+
if imported:
|
|
2751
|
+
return by_args(imported) if len({c.parent_id for c in imported}) == 1 else []
|
|
2752
|
+
if len(free) == 1 and (lang, name) not in self.outside_names:
|
|
2753
|
+
self._guessed = True
|
|
2754
|
+
return list(free)
|
|
2755
|
+
return [] if free else None
|
|
2756
|
+
if r not in ("?", None, "this", "base"):
|
|
2757
|
+
tid = self._generic_var_type(lang, call.src_id, r, call.enclosing_type)
|
|
2758
|
+
if tid == "external":
|
|
2759
|
+
return None
|
|
2760
|
+
if tid:
|
|
2761
|
+
for t in self._chain(tid):
|
|
2762
|
+
found = self.members.get(t, {}).get(name)
|
|
2763
|
+
if found:
|
|
2764
|
+
return by_args(found)
|
|
2765
|
+
if r == "?" and (call.receiver_type or call.chain is not None):
|
|
2766
|
+
# made on `new Foo()` or on what another call returns
|
|
2767
|
+
tid = (self.types_by_name.get((lang, call.receiver_type)) or [None])[0] if call.receiver_type else None
|
|
2768
|
+
if call.chain is not None and not call.receiver_type:
|
|
2769
|
+
rt = self._generic_returns(lang, call.chain)
|
|
2770
|
+
if rt and rt[0] is None and rt[1]:
|
|
2771
|
+
return None # a type from outside the repo
|
|
2772
|
+
tid = rt[0] if rt else None
|
|
2773
|
+
if tid:
|
|
2774
|
+
for t in self._chain(tid):
|
|
2775
|
+
found = self.members.get(t, {}).get(name)
|
|
2776
|
+
if found:
|
|
2777
|
+
return by_args(found)
|
|
2778
|
+
if r == "?":
|
|
2779
|
+
pool, owners = ix["typed"], ix["typed_owners"]
|
|
2780
|
+
if not pool:
|
|
2781
|
+
return None
|
|
2782
|
+
near = self._in_files(ix["typed_file"], near_files, fid)
|
|
2783
|
+
if near and len({c.parent_id for c in near}) == 1:
|
|
2784
|
+
self._guessed = True
|
|
2785
|
+
return by_args(near)
|
|
2786
|
+
else:
|
|
2787
|
+
# The receiver names a type (static call) or a package / module directory (Go's pkg.Func, Rust's mod::f).
|
|
2788
|
+
owned = [c for t in self.types_by_name.get((lang, r), []) for c in self.members.get(t, {}).get(name, [])]
|
|
2789
|
+
if owned:
|
|
2790
|
+
return by_args(owned)
|
|
2791
|
+
# Go's store.New(): the package the file imports under that name, before any other directory called store
|
|
2792
|
+
pkg = [c for c in self._in_files(ix["free_file"], self.import_targets.get(fid, ()))
|
|
2793
|
+
if (self.nodes[self.file_of[c.id]].path or "").rpartition("/")[0].rsplit("/", 1)[-1] == r]
|
|
2794
|
+
if pkg and len({self.file_of[c.id].rpartition("/")[0] for c in pkg}) == 1:
|
|
2795
|
+
return by_args(pkg)
|
|
2796
|
+
if r in ix["free_dir_base"]:
|
|
2797
|
+
return [ix["free_dir_base"][r]]
|
|
2798
|
+
# module.f() reaches the module's own functions before any method of that name in it
|
|
2799
|
+
if r in ix["stem_free"]:
|
|
2800
|
+
return [ix["stem_free"][r]]
|
|
2801
|
+
if r in ix["stem"]:
|
|
2802
|
+
return [ix["stem"][r]]
|
|
2803
|
+
if types and r not in self.types_by_name.get((lang, r), ()) and self._generic_var_type(lang, call.src_id, r, call.enclosing_type) is None:
|
|
2804
|
+
# module.Type(...): a type reached through the module or package the receiver names
|
|
2805
|
+
via = [t for t in types if self.file_of.get(t) in near_files
|
|
2806
|
+
or (self.nodes[self.file_of[t]].path.rpartition("/")[0].rsplit("/", 1)[-1] == r if t in self.file_of else False)]
|
|
2807
|
+
if len(via) == 1:
|
|
2808
|
+
return ctor_of(via)
|
|
2809
|
+
if r[:1].isupper() and not self.types_by_name.get((lang, r)):
|
|
2810
|
+
return None # a type from outside: Foo.bar()
|
|
2811
|
+
pool, owners = cands, ix["owners"]
|
|
2812
|
+
if (lang, name) in self.outside_names or len(name) <= 2:
|
|
2813
|
+
self.stats[BY_LANGUAGE[lang].NAME]["calls_guess_declined"] += 1
|
|
2814
|
+
return None
|
|
2815
|
+
if len(owners) == 1:
|
|
2816
|
+
self._guessed = True
|
|
2817
|
+
self.stats[BY_LANGUAGE[lang].NAME]["calls_by_unique_name"] += 1
|
|
2818
|
+
return list(pool)
|
|
2819
|
+
return []
|
|
2820
|
+
|
|
2821
|
+
def _resolve_call(self, lang: str, fid: str, call: CallSite) -> Optional[list[Node]]:
|
|
2822
|
+
"""None = defined outside the workspace; [] = defined here but not pinned down."""
|
|
2823
|
+
if getattr(BY_LANGUAGE.get(lang), "GENERIC", False):
|
|
2824
|
+
return self._generic_call(lang, fid, call)
|
|
2825
|
+
name, argc = call.name, call.argc
|
|
2826
|
+
self._last_src = call.src_id
|
|
2827
|
+
if lang == "typescript": # one hop is too strict where a store or a module object sits between caller and function
|
|
2828
|
+
reach = self._reach(fid)
|
|
2829
|
+
defined_here = any(self.file_of.get(c.id) in reach for c in self.by_name.get((lang, name), ())) or bool(
|
|
2830
|
+
self.types_by_name.get((lang, name)))
|
|
2831
|
+
else:
|
|
2832
|
+
defined_here = any(self._can_see(fid, c.id) for c in self.by_name.get((lang, name), ())) or (
|
|
2833
|
+
lang in FILE_MODULE and bool(self.types_by_name.get((lang, name))))
|
|
2834
|
+
if name == ".ctor":
|
|
2835
|
+
tid = self._type(lang, call.receiver_type or "", call.src_id)
|
|
2836
|
+
if tid is None:
|
|
2837
|
+
return None
|
|
2838
|
+
ctors = self.members.get(tid, {}).get(".ctor")
|
|
2839
|
+
return self._pick(ctors, argc) if ctors else None
|
|
2840
|
+
if call.receiver is None:
|
|
2841
|
+
# Local functions of the caller, innermost first.
|
|
2842
|
+
cur = call.src_id
|
|
2843
|
+
while cur in self.nodes and self.nodes[cur].kind in ("callable", "test", "type"):
|
|
2844
|
+
if self.nodes[cur].kind == "type":
|
|
2845
|
+
if lang != "python":
|
|
2846
|
+
break
|
|
2847
|
+
cur = self.nodes[cur].parent_id # a method does not see its class's names, but does see the function around the class
|
|
2848
|
+
continue
|
|
2849
|
+
local = self.members.get(cur, {}).get(name)
|
|
2850
|
+
if local:
|
|
2851
|
+
return self._pick(local, argc)
|
|
2852
|
+
if lang == "python" and self.nodes.get(f"{cur}.{name}") is not None and self.nodes[f"{cur}.{name}"].kind == "type":
|
|
2853
|
+
return self._py_symbol(f"{cur}.{name}", argc) or [] # a class defined in the function
|
|
2854
|
+
cur = self.nodes[cur].parent_id
|
|
2855
|
+
if lang in FILE_MODULE:
|
|
2856
|
+
return self._py_bare(fid, call, defined_here)
|
|
2857
|
+
found = self._methods(call.enclosing_type, name, argc)
|
|
2858
|
+
if found:
|
|
2859
|
+
return found
|
|
2860
|
+
for tn in self.cs_static[fid]:
|
|
2861
|
+
found = self._methods(self._type(lang, tn, call.src_id), name, argc)
|
|
2862
|
+
if found:
|
|
2863
|
+
return found
|
|
2864
|
+
return [] if defined_here else None
|
|
2865
|
+
if lang == "typescript" and call.receiver in self.outside_imports[fid]:
|
|
2866
|
+
return None # React.useState(), path.join(): a namespace from outside
|
|
2867
|
+
inner = call.chain
|
|
2868
|
+
if lang == "python" and inner is not None and inner.attr and inner.chain is None and inner.receiver_type is None \
|
|
2869
|
+
and self._py_bound(fid, inner.src_id, inner.receiver or ""):
|
|
2870
|
+
target, symbol = self._py_name(fid, inner.src_id, inner.receiver)
|
|
2871
|
+
sub = self.py_modules[_repo_of(target)].get(f"{self._modpath(target)}.{inner.name}") if target and symbol is None else None
|
|
2872
|
+
if sub:
|
|
2873
|
+
return self._py_symbol(self._py_export(sub, name) or "", argc) # flask.json.dumps(): a submodule's function
|
|
2874
|
+
imported = (call.receiver_type is None and self._py_bound(fid, call.src_id, call.receiver or "")) if lang == "python" \
|
|
2875
|
+
else call.receiver in self.py_names[fid] # a typed local of the same name hides the import
|
|
2876
|
+
if lang in FILE_MODULE and call.receiver not in ("this",) and imported:
|
|
2877
|
+
target, symbol = self._py_name(fid, call.src_id, call.receiver) if lang == "python" else self.py_names[fid][call.receiver]
|
|
2878
|
+
if target is None:
|
|
2879
|
+
return [] if defined_here else None
|
|
2880
|
+
if symbol is None:
|
|
2881
|
+
return self._py_symbol(self._py_export(target, name) or "", argc)
|
|
2882
|
+
tid = self._py_export(target, symbol)
|
|
2883
|
+
if tid is None: # a module passed on under a name: `export * as ns from './format'` in a barrel
|
|
2884
|
+
mod = self._export_module(target, symbol)
|
|
2885
|
+
if mod:
|
|
2886
|
+
return self._py_symbol(self._py_export(mod, name) or "", argc)
|
|
2887
|
+
if tid is None and lang == "python":
|
|
2888
|
+
tid = self._py_var(target, symbol) # `from .globals import g`: a variable of a known type
|
|
2889
|
+
found = self._methods(tid, name, argc)
|
|
2890
|
+
return found or ([] if defined_here else None)
|
|
2891
|
+
tid, known = self._receiver_type(lang, call)
|
|
2892
|
+
if tid and lang == "python" and tid in self.nodes and self.nodes[tid].kind == "callable":
|
|
2893
|
+
return [self.nodes[tid]] if name == "__call__" else None # a function value: only calling it means anything
|
|
2894
|
+
if tid:
|
|
2895
|
+
# `self.go()` in a nested class is not the enclosing class's `go`: a receiver reaches its own type and bases
|
|
2896
|
+
found = self._methods(tid, name, argc, enclosing=False)
|
|
2897
|
+
if found:
|
|
2898
|
+
return found
|
|
2899
|
+
if lang == "csharp":
|
|
2900
|
+
found = self._extensions(lang, fid, tid, name, argc)
|
|
2901
|
+
if found:
|
|
2902
|
+
return found
|
|
2903
|
+
if not (lang == "typescript" and any(name in self.field_names.get(t, ()) for t in self._chain(tid))) and not (
|
|
2904
|
+
lang == "python" and any(self.nodes[t].attrs.get("open_base") for t in self._chain(tid) if t in self.nodes)):
|
|
2905
|
+
return None # the type is ours but the method is inherited from outside
|
|
2906
|
+
# A property that holds a function (`save: (x) => void` in an interface), or a Python class whose base
|
|
2907
|
+
# is made at run time: what is there is not known from the type, so fall through to the unique-name guess.
|
|
2908
|
+
tid, known = None, False
|
|
2909
|
+
if known:
|
|
2910
|
+
return None # receiver has a type that is not in the workspace
|
|
2911
|
+
if lang == "python" and self.outside_imports[fid]:
|
|
2912
|
+
root = call # sqlite3.connect(), os.path.join(): a module from outside
|
|
2913
|
+
while root.receiver == "?" and root.receiver_type is None and root.chain is not None and root.chain.attr:
|
|
2914
|
+
root = root.chain
|
|
2915
|
+
if root.receiver_type is None and root.chain is None and (root.receiver or "") in self.outside_imports[fid] \
|
|
2916
|
+
and not self._py_bound(fid, call.src_id, root.receiver):
|
|
2917
|
+
return None
|
|
2918
|
+
if not defined_here:
|
|
2919
|
+
return None
|
|
2920
|
+
if lang == "python" and name.startswith("__") and name.endswith("__"):
|
|
2921
|
+
return [] # __enter__, __call__: run implicitly on all kinds of objects, never guessed by name
|
|
2922
|
+
if (lang, name) in self.outside_names:
|
|
2923
|
+
self.stats[BY_LANGUAGE[lang].NAME]["calls_guess_declined"] += 1
|
|
2924
|
+
# In TypeScript most receivers are untyped, and a name seen on built-in types (push, get, keys) is
|
|
2925
|
+
# far more often the built-in than the one function here that shares it.
|
|
2926
|
+
return None if lang == "typescript" else []
|
|
2927
|
+
self.stats[BY_LANGUAGE[lang].NAME]["calls_by_unique_name"] += 1
|
|
2928
|
+
self._guessed = True
|
|
2929
|
+
# Receiver type unknown: accept only a name that is defined exactly once.
|
|
2930
|
+
cands = self._pick([c for c in self.by_name[(lang, name)] if (
|
|
2931
|
+
self.file_of.get(c.id) in self._reach(fid) if lang == "typescript" else self._can_see(fid, c.id))], argc)
|
|
2932
|
+
if lang == "python":
|
|
2933
|
+
# `con.close()` is a method: not a module's function (the receiver is no module), nor one nested in another
|
|
2934
|
+
cands = [c for c in cands if c.parent_id in self.nodes and self.nodes[c.parent_id].kind == "type"]
|
|
2935
|
+
owners = {c.parent_id for c in cands}
|
|
2936
|
+
if len(owners) > 1:
|
|
2937
|
+
# Several declarations: if all but one are overrides or implementations of the same
|
|
2938
|
+
# root declaration, the call is to that root (virtual dispatch).
|
|
2939
|
+
roots = {o for o in owners if not any(b in owners for b in self._chain(o)[1:])}
|
|
2940
|
+
if len(roots) == 1 and all(next(iter(roots)) in self._chain(o) for o in owners):
|
|
2941
|
+
root = roots.pop()
|
|
2942
|
+
return [c for c in cands if c.parent_id == root]
|
|
2943
|
+
return []
|
|
2944
|
+
return cands
|
|
2945
|
+
|
|
2946
|
+
def _resolve_overrides(self) -> None:
|
|
2947
|
+
"""Link each method to the base or interface method it implements, by name and arity."""
|
|
2948
|
+
self.implementers: dict[str, list[str]] = defaultdict(list)
|
|
2949
|
+
linked = set()
|
|
2950
|
+
|
|
2951
|
+
def link(impl_id: str, base_method: str) -> None:
|
|
2952
|
+
if (impl_id, base_method) not in linked:
|
|
2953
|
+
linked.add((impl_id, base_method))
|
|
2954
|
+
self.edges.append(Edge("overrides", impl_id, base_method, "heuristic"))
|
|
2955
|
+
self.implementers[base_method].append(impl_id)
|
|
2956
|
+
for tid, bases in list(self.bases.items()):
|
|
2957
|
+
if not bases:
|
|
2958
|
+
continue
|
|
2959
|
+
chain = [base for base in self._chain(tid, enclosing=False)[1:]
|
|
2960
|
+
if self.nodes[base].kind == "type" and base != self.nodes[tid].parent_id]
|
|
2961
|
+
python = self.nodes[tid].language == "python"
|
|
2962
|
+
for name, impls in self.members.get(tid, {}).items():
|
|
2963
|
+
if name in (".ctor", ".dtor"):
|
|
2964
|
+
continue
|
|
2965
|
+
for impl in impls:
|
|
2966
|
+
if impl.kind != "callable":
|
|
2967
|
+
continue
|
|
2968
|
+
for base in chain:
|
|
2969
|
+
# Python and TypeScript have no overloads at run time: a method replaces the base's of that name,
|
|
2970
|
+
# whatever it takes
|
|
2971
|
+
cands = [c for c in self.members.get(base, {}).get(name, [])
|
|
2972
|
+
if impl.language in ("python", "typescript") or c.attrs.get("argc_max") == impl.attrs.get("argc_max")]
|
|
2973
|
+
if cands:
|
|
2974
|
+
link(impl.id, cands[0].id)
|
|
2975
|
+
# With several bases (a mixin and a base class), a Python method replaces the one
|
|
2976
|
+
# each of them declares: a call on the base class's `self` runs it too.
|
|
2977
|
+
if not python:
|
|
2978
|
+
break
|
|
2979
|
+
if not python:
|
|
2980
|
+
continue
|
|
2981
|
+
# A name the class inherits from a mixin shadows the same name on a later base: for a call from that
|
|
2982
|
+
# base's own code, the mixin's method is the one that runs.
|
|
2983
|
+
declared: dict[str, list[Node]] = defaultdict(list)
|
|
2984
|
+
for base in chain:
|
|
2985
|
+
for name, cands in self.members.get(base, {}).items():
|
|
2986
|
+
if name not in (".ctor", ".dtor") and name not in self.members.get(tid, {}):
|
|
2987
|
+
declared[name].extend(c for c in cands if c.kind == "callable")
|
|
2988
|
+
for name, methods in declared.items():
|
|
2989
|
+
for later in methods[1:]:
|
|
2990
|
+
if later.parent_id != methods[0].parent_id:
|
|
2991
|
+
link(methods[0].id, later.id)
|
|
2992
|
+
|
|
2993
|
+
# -- channels ------------------------------------------------------------
|
|
2994
|
+
def _event_field(self, type_id: Optional[str], name: str) -> Optional[str]:
|
|
2995
|
+
for t in self._chain(type_id):
|
|
2996
|
+
fid = f"{t}.{name}"
|
|
2997
|
+
n = self.nodes.get(fid)
|
|
2998
|
+
if n is not None and n.kind == "field" and n.attrs.get("native_kind") == "event":
|
|
2999
|
+
return fid
|
|
3000
|
+
return None
|
|
3001
|
+
|
|
3002
|
+
def _resolve_fields(self) -> None:
|
|
3003
|
+
"""Link each function to the fields it reads and assigns."""
|
|
3004
|
+
field_id: dict[str, dict[str, str]] = defaultdict(dict)
|
|
3005
|
+
by_name: dict[tuple, list[str]] = defaultdict(list)
|
|
3006
|
+
for n in self.nodes.values():
|
|
3007
|
+
if n.kind == "field" and n.attrs.get("native_kind") not in ("enum_member", "event"):
|
|
3008
|
+
field_id[n.parent_id][n.name] = n.id
|
|
3009
|
+
by_name[(n.language, n.name)].append(n.id)
|
|
3010
|
+
found: dict[tuple, list] = {}
|
|
3011
|
+
for fid, res in self.results.items():
|
|
3012
|
+
lang = self.file_lang[fid]
|
|
3013
|
+
st = self.stats[BY_LANGUAGE[lang].NAME]
|
|
3014
|
+
for u in self._open(fid, res).field_uses:
|
|
3015
|
+
if u.src_id not in self.nodes:
|
|
3016
|
+
continue
|
|
3017
|
+
target, guessed = None, False
|
|
3018
|
+
if u.receiver in (None, "this"):
|
|
3019
|
+
if u.receiver is None and lang in FILE_MODULE:
|
|
3020
|
+
continue # a bare name in Python is a local or a global, never an attribute
|
|
3021
|
+
owners = self._chain(u.enclosing_type)
|
|
3022
|
+
elif u.receiver == "base":
|
|
3023
|
+
owners = self._chain(u.enclosing_type)[1:]
|
|
3024
|
+
else:
|
|
3025
|
+
probe = CallSite(u.src_id, u.name, u.receiver, u.receiver_type, 0, u.line, u.enclosing_type, chain=u.chain)
|
|
3026
|
+
tid, known = self._receiver_type(lang, probe)
|
|
3027
|
+
owners = self._chain(tid) if tid else []
|
|
3028
|
+
if lang == "csharp" and not tid and not known and len(u.name) > 3:
|
|
3029
|
+
# Receiver type unknown: accept only a field name declared exactly once, and say it is a guess.
|
|
3030
|
+
cands = [c for c in by_name.get((lang, u.name), ()) if self._can_see(fid, c)]
|
|
3031
|
+
if len(cands) == 1 and (lang, u.name) not in self.outside_names:
|
|
3032
|
+
target, guessed = cands[0], True
|
|
3033
|
+
for t in owners:
|
|
3034
|
+
if u.name in field_id.get(t, ()):
|
|
3035
|
+
target = field_id[t][u.name]
|
|
3036
|
+
break
|
|
3037
|
+
if lang == "python" and u.access != "w" and owners:
|
|
3038
|
+
self._read_getters(u, owners, target)
|
|
3039
|
+
if target is None or target == u.src_id:
|
|
3040
|
+
continue
|
|
3041
|
+
st["field_uses"] += 1
|
|
3042
|
+
for kind in (("reads",) if u.access == "r" else ("writes",) if u.access in ("w", "i") else ("reads", "writes")):
|
|
3043
|
+
slot = found.setdefault((kind, u.src_id, target), [0, u.line, False, 0])
|
|
3044
|
+
slot[0] += 1
|
|
3045
|
+
slot[1] = min(slot[1], u.line)
|
|
3046
|
+
slot[2] = slot[2] or guessed
|
|
3047
|
+
slot[3] += u.access == "i"
|
|
3048
|
+
self._shut()
|
|
3049
|
+
for (kind, src, dst), (n, line, guessed, init) in found.items():
|
|
3050
|
+
attrs = {"n": n, "line": line}
|
|
3051
|
+
if init == n:
|
|
3052
|
+
attrs["init"] = True # only ever set while creating the object, never changed afterwards
|
|
3053
|
+
self.edges.append(Edge(kind, src, dst, "guess" if guessed else "heuristic", attrs))
|
|
3054
|
+
|
|
3055
|
+
def _is_getter(self, fn: Node) -> bool:
|
|
3056
|
+
"""A Python method that runs when its attribute is read: @property, or a decorator that is a descriptor
|
|
3057
|
+
class (one defining __get__, such as werkzeug's cached_property) declared somewhere in the workspace."""
|
|
3058
|
+
if fn.kind != "callable" or not fn.attrs.get("decorators"):
|
|
3059
|
+
return False
|
|
3060
|
+
if fn.id not in self._getter_memo:
|
|
3061
|
+
hit = False
|
|
3062
|
+
for d in fn.attrs["decorators"]:
|
|
3063
|
+
d = d.lstrip("@").split("(", 1)[0].strip()
|
|
3064
|
+
if d in ("property", "functools.cached_property", "abc.abstractproperty"):
|
|
3065
|
+
hit = True
|
|
3066
|
+
elif re.fullmatch(r"[A-Za-z_][\w.]*", d) and not d.endswith((".setter", ".deleter")):
|
|
3067
|
+
tid = self._type("python", d.rsplit(".", 1)[-1], fn.id)
|
|
3068
|
+
hit = bool(tid and any("__get__" in self.members.get(t, {}) for t in self._chain(tid)))
|
|
3069
|
+
if hit:
|
|
3070
|
+
break
|
|
3071
|
+
self._getter_memo[fn.id] = hit
|
|
3072
|
+
return self._getter_memo[fn.id]
|
|
3073
|
+
|
|
3074
|
+
def _read_getters(self, u: FieldUse, owners: list[str], field: Optional[str]) -> None:
|
|
3075
|
+
"""Reading an attribute that a getter computes runs the getter, so the read is a call. On `self`, a
|
|
3076
|
+
subclass may compute an attribute the class itself holds as a plain value or a getter of its own
|
|
3077
|
+
(Flask's Request.max_content_length over Werkzeug's): those getters are reached too, as dispatch."""
|
|
3078
|
+
if not hasattr(self, "_getter_memo"):
|
|
3079
|
+
self._getter_memo: dict[str, bool] = {}
|
|
3080
|
+
self._subtypes: dict[str, set] = defaultdict(set)
|
|
3081
|
+
for sub, bases in self.bases.items():
|
|
3082
|
+
for b in bases:
|
|
3083
|
+
self._subtypes[b].add(sub)
|
|
3084
|
+
hits = []
|
|
3085
|
+
for t in owners:
|
|
3086
|
+
if field is not None and self.nodes[field].parent_id == t:
|
|
3087
|
+
break # a plain value nearer than any getter
|
|
3088
|
+
found = [m for m in self.members.get(t, {}).get(u.name, []) if self._is_getter(m)]
|
|
3089
|
+
if found:
|
|
3090
|
+
hits.append((found[0], "property"))
|
|
3091
|
+
break
|
|
3092
|
+
if u.receiver == "this" and (hits or field is not None):
|
|
3093
|
+
seen, queue = set(), [owners[0]]
|
|
3094
|
+
while queue:
|
|
3095
|
+
for sub in self._subtypes.get(queue.pop(), ()):
|
|
3096
|
+
if sub not in seen:
|
|
3097
|
+
seen.add(sub)
|
|
3098
|
+
queue.append(sub)
|
|
3099
|
+
hits += [(m, "virtual") for m in self.members.get(sub, {}).get(u.name, [])[:1] if self._is_getter(m)]
|
|
3100
|
+
for fn, dispatch in hits:
|
|
3101
|
+
if fn.id != u.src_id:
|
|
3102
|
+
self.calls.append((u.src_id, fn.id, dispatch, "heuristic", u.line))
|
|
3103
|
+
|
|
3104
|
+
def _resolve_events(self) -> None:
|
|
3105
|
+
"""Link the code that raises an event to the code that handles it."""
|
|
3106
|
+
raisers: dict[str, list] = defaultdict(list)
|
|
3107
|
+
handlers: dict[str, list] = defaultdict(list)
|
|
3108
|
+
st = self.channel_stats["event"]
|
|
3109
|
+
for fid, res in self.results.items():
|
|
3110
|
+
lang = self.file_lang[fid]
|
|
3111
|
+
for ev in res.events:
|
|
3112
|
+
if ev.kind == "raise":
|
|
3113
|
+
field = self._event_field(ev.enclosing_type, ev.event)
|
|
3114
|
+
if field:
|
|
3115
|
+
raisers[field].append((ev.src_id, ev.line))
|
|
3116
|
+
continue
|
|
3117
|
+
if ev.receiver is None:
|
|
3118
|
+
field = self._event_field(ev.enclosing_type, ev.event)
|
|
3119
|
+
else:
|
|
3120
|
+
probe = CallSite(ev.src_id, ev.event, ev.receiver, ev.receiver_type, 0, ev.line, ev.enclosing_type)
|
|
3121
|
+
tid, _ = self._receiver_type(lang, probe)
|
|
3122
|
+
field = self._event_field(tid, ev.event) if tid else None
|
|
3123
|
+
if not field:
|
|
3124
|
+
if ev.handler is None or ev.receiver is not None:
|
|
3125
|
+
st["subscriptions_to_outside_events"] += 1
|
|
3126
|
+
continue
|
|
3127
|
+
target, via = ev.src_id, "lambda"
|
|
3128
|
+
if ev.handler:
|
|
3129
|
+
found = self._methods(ev.enclosing_type, ev.handler, 1) or self._methods(ev.enclosing_type, ev.handler, 0)
|
|
3130
|
+
if found:
|
|
3131
|
+
target, via = found[0].id, "method"
|
|
3132
|
+
handlers[field].append((target, via, ev.line, ev.src_id))
|
|
3133
|
+
seen = set()
|
|
3134
|
+
for field, subs in handlers.items():
|
|
3135
|
+
st["events_with_handlers"] += 1
|
|
3136
|
+
for raiser, _line in raisers.get(field, []):
|
|
3137
|
+
for target, via, line, subscriber in subs:
|
|
3138
|
+
key = (raiser, target, field)
|
|
3139
|
+
if key in seen or raiser not in self.nodes or target not in self.nodes:
|
|
3140
|
+
continue
|
|
3141
|
+
seen.add(key)
|
|
3142
|
+
st["links"] += 1
|
|
3143
|
+
self.edges.append(Edge("communicates", raiser, target, "heuristic", {
|
|
3144
|
+
"channel": "event", "address": field, "direction": "push", "handler": via,
|
|
3145
|
+
"subscriber": subscriber, "subscribed_at": line}))
|
|
3146
|
+
if field not in raisers:
|
|
3147
|
+
st["events_never_raised_here"] += 1
|
|
3148
|
+
st["events_declared"] = sum(1 for n in self.nodes.values()
|
|
3149
|
+
if n.kind == "field" and n.attrs.get("native_kind") == "event")
|
|
3150
|
+
|
|
3151
|
+
GENERIC_DIRS = {"data", "temp", "file", "files", "json", "path", "output", "outputs", "input", "home", "user", "users",
|
|
3152
|
+
"test", "tests", "docs", "static", "assets", "resources", "config", "build", "dist", "local"}
|
|
3153
|
+
|
|
3154
|
+
def _route_handlers(self, routes: list) -> tuple[list, dict]:
|
|
3155
|
+
"""The functions that answer routes. An inline handler is its own function (an adapter made it, nested in
|
|
3156
|
+
the function that registers the route): the registrar registers it, which the map keeps as a call so that
|
|
3157
|
+
whatever runs the registrar still reaches the handler. A handler given by name (server.get("/x", list))
|
|
3158
|
+
is the function that name refers to, when the registrar's reference to it was resolved.
|
|
3159
|
+
Returns the routes with their serving end moved to the handler, and each handler's registrar."""
|
|
3160
|
+
handlers: dict[str, str] = {}
|
|
3161
|
+
for n in list(self.nodes.values()):
|
|
3162
|
+
if n.kind == "callable" and n.attrs.get("native_kind") == "route_handler" and n.parent_id in self.nodes:
|
|
3163
|
+
handlers[n.id] = n.parent_id
|
|
3164
|
+
self.calls.append((n.parent_id, n.id, "registers", "heuristic", n.attrs.get("registered_at") or n.span_start))
|
|
3165
|
+
named = [i for i, (e, _) in enumerate(routes) if e.handler]
|
|
3166
|
+
if named:
|
|
3167
|
+
refs: dict[str, set] = defaultdict(set)
|
|
3168
|
+
wanted = {routes[i][0].src_id for i in named}
|
|
3169
|
+
for src, dst, dispatch, _p, _l in self.calls:
|
|
3170
|
+
if dispatch == "reference" and src in wanted:
|
|
3171
|
+
refs[src].add(dst)
|
|
3172
|
+
routes = list(routes)
|
|
3173
|
+
for i in named:
|
|
3174
|
+
e, res = routes[i]
|
|
3175
|
+
name = e.handler.rsplit(".", 1)[-1]
|
|
3176
|
+
hits = {d for d in refs.get(e.src_id, ()) if d in self.nodes and self.nodes[d].name == name}
|
|
3177
|
+
if len(hits) == 1:
|
|
3178
|
+
target = hits.pop()
|
|
3179
|
+
handlers.setdefault(target, e.src_id)
|
|
3180
|
+
routes[i] = (Endpoint(e.channel, e.role, target, e.address, e.line, e.method, e.literals, e.handler), res)
|
|
3181
|
+
return routes, handlers
|
|
3182
|
+
|
|
3183
|
+
@staticmethod
|
|
3184
|
+
def _one_handler(cands: list, handlers: dict, method: Optional[str], literal) -> list:
|
|
3185
|
+
"""A request that several handlers' routes fit. A route that names more of the path outright wins
|
|
3186
|
+
(/api/review/health over /api/review/:id), as routers pick it; then, for a request whose method is not
|
|
3187
|
+
written, a GET, which is what a request sent with no method is; failing both, the function that registers
|
|
3188
|
+
them all answers it, as it did before handlers had functions of their own. Empty when none settles it."""
|
|
3189
|
+
most = max(literal(r) for r in cands)
|
|
3190
|
+
cands = [r for r in cands if literal(r) == most]
|
|
3191
|
+
if len({r.src_id for r in cands}) == 1:
|
|
3192
|
+
return cands
|
|
3193
|
+
gets = [r for r in cands if r.method in (None, "GET")] if method is None else []
|
|
3194
|
+
if gets and len({r.src_id for r in gets}) == 1:
|
|
3195
|
+
return gets
|
|
3196
|
+
owners = {handlers.get(r.src_id) for r in cands}
|
|
3197
|
+
if len(owners) == 1 and None not in owners:
|
|
3198
|
+
r, same = cands[0], {c.method for c in cands}
|
|
3199
|
+
return [Endpoint(r.channel, r.role, owners.pop(), r.address, r.line, same.pop() if len(same) == 1 else None,
|
|
3200
|
+
r.literals)]
|
|
3201
|
+
return []
|
|
3202
|
+
|
|
3203
|
+
def _resolve_endpoints(self) -> None:
|
|
3204
|
+
"""Link the two ends of channels that are not calls: an HTTP request to its route, and code that
|
|
3205
|
+
writes a file to code that reads it."""
|
|
3206
|
+
http, files = self.channel_stats["http"], self.channel_stats["file"]
|
|
3207
|
+
|
|
3208
|
+
def root_of(i): # the outermost function a nested function or inline test sits in
|
|
3209
|
+
cur = i
|
|
3210
|
+
while self.nodes[cur].parent_id in self.nodes and self.nodes[self.nodes[cur].parent_id].kind in ("callable", "test"):
|
|
3211
|
+
cur = self.nodes[cur].parent_id
|
|
3212
|
+
return cur
|
|
3213
|
+
|
|
3214
|
+
def segments(path):
|
|
3215
|
+
path = re.sub(r"^[a-z]+://[^/]+", "", path).split("?", 1)[0].split("#", 1)[0]
|
|
3216
|
+
return [x for x in path.split("/") if x]
|
|
3217
|
+
routes, requests, io = [], [], []
|
|
3218
|
+
for res in self.results.values():
|
|
3219
|
+
for e in res.endpoints:
|
|
3220
|
+
if e.src_id not in self.nodes or e.channel not in ("http", "file"):
|
|
3221
|
+
continue # the other channels are linked in leyline.channels
|
|
3222
|
+
(routes if (e.channel, e.role) == ("http", "serve") else requests if e.channel == "http" else io).append((e, res))
|
|
3223
|
+
http["routes"], http["requests"] = len(routes), sum(1 for e, _ in requests if e.role != "maybe")
|
|
3224
|
+
routes, handlers = self._route_handlers(routes)
|
|
3225
|
+
|
|
3226
|
+
def fits(route, request):
|
|
3227
|
+
a, b = segments(route.address), segments(request.address)
|
|
3228
|
+
if route.method and request.method and route.method != request.method:
|
|
3229
|
+
return False
|
|
3230
|
+
if a and _wildcard(a[-1]):
|
|
3231
|
+
return _fits_wildcard(a, b)
|
|
3232
|
+
if len(a) != len(b):
|
|
3233
|
+
return False
|
|
3234
|
+
# ASP.NET's [controller] is the class's name, which clients write in lower case
|
|
3235
|
+
return all(_segment_fits(x, y) for x, y in zip(a, b))
|
|
3236
|
+
seen = set()
|
|
3237
|
+
for req, _ in requests:
|
|
3238
|
+
cands = [r for r, _ in routes if fits(r, req)]
|
|
3239
|
+
# In a workspace, a route in the request's own repository wins over one in another.
|
|
3240
|
+
cands = [r for r in cands if _repo_of(r.src_id) == _repo_of(req.src_id)] or cands
|
|
3241
|
+
near = [r for r in cands if root_of(r.src_id) == root_of(req.src_id)] # a route declared inside the same test
|
|
3242
|
+
chosen = near or (cands if len({r.src_id for r in cands}) == 1 else [])
|
|
3243
|
+
literal = lambda r: sum(1 for x in segments(r.address) if not x.startswith(("<", "{", ":")) and not _wildcard(x))
|
|
3244
|
+
if handlers and len({r.src_id for r in chosen or cands}) > 1: # several handlers, perhaps of one registrar
|
|
3245
|
+
chosen = self._one_handler(chosen or cands, handlers, req.method, literal) or chosen
|
|
3246
|
+
elif len({r.src_id for r in chosen or cands}) > 1:
|
|
3247
|
+
# In every language, the route that names more of the path outright wins: /items/special over /items/{id}.
|
|
3248
|
+
most = max(literal(r) for r in chosen or cands)
|
|
3249
|
+
top = [r for r in chosen or cands if literal(r) == most]
|
|
3250
|
+
chosen = top if len({r.src_id for r in top}) == 1 else chosen
|
|
3251
|
+
if req.role == "maybe": # a path handed to a wrapper: a request only when exactly one route serves it
|
|
3252
|
+
chosen = chosen if len(segments(req.address)) >= 2 and len({r.src_id for r in chosen}) == 1 else []
|
|
3253
|
+
elif cands and not chosen:
|
|
3254
|
+
http["ambiguous"] += 1
|
|
3255
|
+
for r in chosen:
|
|
3256
|
+
# one link per route called: a function that calls several routes of one handler (a minimal API's
|
|
3257
|
+
# registrar, a router) is linked for each of them
|
|
3258
|
+
key = (req.src_id, r.src_id, r.address, req.method or r.method)
|
|
3259
|
+
if key not in seen and req.src_id != r.src_id:
|
|
3260
|
+
seen.add(key)
|
|
3261
|
+
http["links"] += 1
|
|
3262
|
+
self.edges.append(Edge("communicates", req.src_id, r.src_id, "heuristic",
|
|
3263
|
+
{"channel": "http", "address": f"{req.method or r.method or 'ANY'} {r.address}", "line": req.line,
|
|
3264
|
+
**({"handler": True} if r.src_id in handlers else {})}))
|
|
3265
|
+
|
|
3266
|
+
# Files: a writer and a reader are linked when the path fragments written in each agree.
|
|
3267
|
+
def tokens(strings):
|
|
3268
|
+
dirs, exts, names = set(), set(), set()
|
|
3269
|
+
for s in strings:
|
|
3270
|
+
s = s.replace("\\", "/")
|
|
3271
|
+
if len(s) > 160 or any(ch.isspace() for ch in s):
|
|
3272
|
+
continue
|
|
3273
|
+
parts = [x for x in s.split("/") if x not in ("", ".", "..")]
|
|
3274
|
+
for i, seg in enumerate(parts):
|
|
3275
|
+
clean = re.sub(r"\{[^}]*\}|\*", "", seg)
|
|
3276
|
+
if i == len(parts) - 1 and "." in seg:
|
|
3277
|
+
stem, _, ext = clean.partition(".")
|
|
3278
|
+
if re.fullmatch(r"[A-Za-z0-9_]{1,10}(\.[A-Za-z0-9_]{1,10})?", ext or ""):
|
|
3279
|
+
exts.add("." + ext.lower())
|
|
3280
|
+
if stem and clean == seg and len(stem) > 2:
|
|
3281
|
+
names.add(seg.lower())
|
|
3282
|
+
elif re.fullmatch(r"[A-Za-z][A-Za-z0-9_\-]{3,}", seg) and seg.lower() not in self.GENERIC_DIRS:
|
|
3283
|
+
dirs.add(seg.lower())
|
|
3284
|
+
return dirs, exts, names
|
|
3285
|
+
by_fn: dict[tuple, list] = defaultdict(list)
|
|
3286
|
+
for e, res in io:
|
|
3287
|
+
by_fn[(e.src_id, e.role)].extend(e.literals + res.path_strings.get(e.src_id, []) + res.path_strings.get(root_of(e.src_id), []))
|
|
3288
|
+
files["read_sites"] = sum(1 for e, _ in io if e.role == "read")
|
|
3289
|
+
files["write_sites"] = sum(1 for e, _ in io if e.role == "write")
|
|
3290
|
+
toks = {k: tokens(v) for k, v in by_fn.items()}
|
|
3291
|
+
pairs = []
|
|
3292
|
+
for (w, wr), (wd, we, wn) in toks.items():
|
|
3293
|
+
if wr != "write":
|
|
3294
|
+
continue
|
|
3295
|
+
for (r, rr), (rd, re_, rn) in toks.items():
|
|
3296
|
+
if rr != "read" or root_of(r) == root_of(w) or _repo_of(r) != _repo_of(w):
|
|
3297
|
+
continue # file paths are matched within one repository; across two they say too little
|
|
3298
|
+
shared_name = wn & rn
|
|
3299
|
+
shared = (wd & rd, we & re_)
|
|
3300
|
+
if shared_name:
|
|
3301
|
+
pairs.append((w, r, sorted(shared_name)[0]))
|
|
3302
|
+
elif shared[0] and shared[1]:
|
|
3303
|
+
pairs.append((w, r, f"{sorted(shared[0], key=lambda x: (-len(x), x))[0]}/*{sorted(shared[1], key=lambda x: (-len(x), x))[0]}"))
|
|
3304
|
+
fan = Counter(p[2] for p in pairs)
|
|
3305
|
+
for w, r, address in pairs:
|
|
3306
|
+
if fan[address] > 8:
|
|
3307
|
+
files["too_common_to_link"] += 1
|
|
3308
|
+
continue # a fragment this common says nothing about who reads whose file
|
|
3309
|
+
files["links"] += 1
|
|
3310
|
+
self.edges.append(Edge("communicates", w, r, "guess", {"channel": "file", "address": address}))
|
|
3311
|
+
|
|
3312
|
+
def _resolve_spawns(self) -> None:
|
|
3313
|
+
"""Link code that launches a program to that program's entry point, when it is in the workspace."""
|
|
3314
|
+
st = self.channel_stats["process"]
|
|
3315
|
+
entries: dict[str, list[str]] = defaultdict(list) # module -> cli entry callables
|
|
3316
|
+
for e in self.edges:
|
|
3317
|
+
if e.kind == "exposes" and self.nodes[e.src_id].attrs.get("trigger") == "cli":
|
|
3318
|
+
mod = self.nodes[self.file_of[e.dst_id]].parent_id if e.dst_id in self.file_of else None
|
|
3319
|
+
if mod:
|
|
3320
|
+
entries[mod].append(e.dst_id)
|
|
3321
|
+
files_by_path = {(_repo_of(n.id), n.path): n.id for n in self.nodes.values() if n.kind == "file"}
|
|
3322
|
+
modules = [(n.path, n.id) for n in self.nodes.values() if n.kind == "module" and n.path]
|
|
3323
|
+
|
|
3324
|
+
def match(text: str, repo: str):
|
|
3325
|
+
text = text.strip().replace("\\", "/")
|
|
3326
|
+
if (repo, text) in files_by_path: # a script path, in the launching file's repository
|
|
3327
|
+
fid = files_by_path[(repo, text)]
|
|
3328
|
+
tops = [e.dst_id for e in self.edges if e.kind == "exposes" and self.file_of.get(e.dst_id) == fid]
|
|
3329
|
+
return tops[0] if tops else fid
|
|
3330
|
+
built = re.match(r"^(.*?/)?(?:dist|build|out|lib)/(.+)\.[cm]?js$", text)
|
|
3331
|
+
if built: # a built script (host/dist/index.js) runs the source it was built from (host/src/index.ts)
|
|
3332
|
+
for src in ("src/", ""):
|
|
3333
|
+
for ext in (".ts", ".mts", ".cts", ".tsx", ".js", ".mjs"):
|
|
3334
|
+
source = f"{built.group(1) or ''}{src}{built.group(2)}{ext}"
|
|
3335
|
+
if (repo, source) in files_by_path:
|
|
3336
|
+
return match(source, repo)
|
|
3337
|
+
for path, mid in sorted(modules, key=lambda m: _repo_of(m[1]) != repo): # own repository first
|
|
3338
|
+
stem = text.rsplit("/", 1)[-1].rsplit(".", 1)[0] if "." in text.rsplit("/", 1)[-1] else None
|
|
3339
|
+
if text == path or text.startswith(path + "/") or (stem and stem == path.rsplit("/", 1)[-1] and text.endswith((".dll", ".exe", ".csproj"))):
|
|
3340
|
+
if not entries.get(mid):
|
|
3341
|
+
continue
|
|
3342
|
+
return entries[mid][0] if len(entries[mid]) == 1 else mid
|
|
3343
|
+
return None
|
|
3344
|
+
|
|
3345
|
+
seen = set()
|
|
3346
|
+
for fid, res in self.results.items():
|
|
3347
|
+
for sp in res.spawns:
|
|
3348
|
+
st["launch_sites"] += 1
|
|
3349
|
+
hits = [(match(s, _repo_of(fid)), s, "heuristic") for s in sp.strings]
|
|
3350
|
+
hits = [h for h in hits if h[0]]
|
|
3351
|
+
if not hits:
|
|
3352
|
+
hits = [(match(s, _repo_of(fid)), s, "guess") for s in dict.fromkeys(sp.file_strings)]
|
|
3353
|
+
hits = [h for h in hits if h[0]]
|
|
3354
|
+
if not hits:
|
|
3355
|
+
st["launches_of_outside_programs"] += 1
|
|
3356
|
+
continue
|
|
3357
|
+
for target, text, precision in dict.fromkeys(hits):
|
|
3358
|
+
if (sp.src_id, target) in seen or target == sp.src_id:
|
|
3359
|
+
continue
|
|
3360
|
+
seen.add((sp.src_id, target))
|
|
3361
|
+
st["links"] += 1
|
|
3362
|
+
self.edges.append(Edge("communicates", sp.src_id, target, precision, {
|
|
3363
|
+
"channel": "process", "address": text,
|
|
3364
|
+
"direction": "both" if sp.pipes else "start", "pipes": sp.pipes, "launched_at": sp.line}))
|
|
3365
|
+
|
|
3366
|
+
# -- flows ---------------------------------------------------------------
|
|
3367
|
+
def _dispatch_reaches(self, home: str, impl: str) -> bool:
|
|
3368
|
+
"""Can a program whose entry is in `home` be running this implementation of an interface?"""
|
|
3369
|
+
if self._visible(home) is not None:
|
|
3370
|
+
return self._can_see(home, impl)
|
|
3371
|
+
# A loose file with no project: it can only be composed of the modules its own module imports.
|
|
3372
|
+
mod = self.nodes[home].parent_id
|
|
3373
|
+
if mod not in self._loose_reach:
|
|
3374
|
+
files = {f for f in self.file_lang if self.nodes[f].parent_id == mod}
|
|
3375
|
+
self._loose_reach[mod] = {mod} | {e.dst_id for e in self.edges if e.kind == "imports" and e.src_id in files}
|
|
3376
|
+
target = self.file_of.get(impl)
|
|
3377
|
+
return target is not None and self.nodes[target].parent_id in self._loose_reach[mod]
|
|
3378
|
+
|
|
3379
|
+
def _build_flows(self, max_depth: int = 8, max_steps: int = 300) -> None:
|
|
3380
|
+
"""Walk the call graph from every entry point and test, in source order."""
|
|
3381
|
+
out: dict[str, list] = defaultdict(list)
|
|
3382
|
+
for src, dst, _disp, _prec, line in self.calls:
|
|
3383
|
+
out[src].append(((line or 0) + self.call_col.get((src, dst, line), 0) / 10000, dst, "calls", None))
|
|
3384
|
+
if not self.keep_results:
|
|
3385
|
+
self.call_col.clear() # read only here (an entry per call): the walk's steps reuse its memory
|
|
3386
|
+
for e in self.edges:
|
|
3387
|
+
if e.kind == "communicates":
|
|
3388
|
+
a = e.attrs or {}
|
|
3389
|
+
if a.get("channel") in ("file", "db", "format"):
|
|
3390
|
+
continue # writing a file, a row or a key does not run whoever reads it later
|
|
3391
|
+
# A registered implementation is listed ahead of the other implementations of its interface.
|
|
3392
|
+
order = 10 ** 9 - 2 if a.get("channel") == "di" and "registered_in" in a else 10 ** 9
|
|
3393
|
+
out[e.src_id].append((a.get("launched_at") or a.get("line") or order, e.dst_id, a.get("channel", "channel"),
|
|
3394
|
+
a.get("subscriber")))
|
|
3395
|
+
# A call to an interface or base method may land in any implementation.
|
|
3396
|
+
for base, impls in self.implementers.items():
|
|
3397
|
+
for impl in impls:
|
|
3398
|
+
out[base].append((10 ** 9 - 1, impl, "dispatch", None))
|
|
3399
|
+
# A test declared inline runs where its runner call sits.
|
|
3400
|
+
for n in self.nodes.values():
|
|
3401
|
+
if n.kind == "test" and n.parent_id:
|
|
3402
|
+
out[n.parent_id].append((n.span_start or 0, n.id, "runs", None))
|
|
3403
|
+
for lst in out.values():
|
|
3404
|
+
lst.sort(key=lambda x: (x[0], x[1]))
|
|
3405
|
+
starts = []
|
|
3406
|
+
for e in self.edges:
|
|
3407
|
+
if e.kind == "exposes":
|
|
3408
|
+
ep = self.nodes[e.src_id]
|
|
3409
|
+
starts.append((e.dst_id, "entry", ep.attrs.get("trigger")))
|
|
3410
|
+
for n in self.nodes.values():
|
|
3411
|
+
if n.kind == "test" or n.attrs.get("is_test"):
|
|
3412
|
+
starts.append((n.id, "test", n.attrs.get("framework")))
|
|
3413
|
+
starts = flow_ids(starts)
|
|
3414
|
+
keep = self._flow_select(out, starts)
|
|
3415
|
+
for start, kind, detail, fid in starts:
|
|
3416
|
+
if start not in self.nodes or (keep is not None and start not in keep):
|
|
3417
|
+
continue
|
|
3418
|
+
entry_file = self.file_of.get(start)
|
|
3419
|
+
seen, steps, truncated = {start}, [(0, 0, start, "start", None, None)], False
|
|
3420
|
+
stack = [(start, 0, 0)]
|
|
3421
|
+
# Depth-first, pre-order, each callable listed once per flow.
|
|
3422
|
+
def walk(node, depth, parent_seq, home):
|
|
3423
|
+
nonlocal truncated
|
|
3424
|
+
if depth >= max_depth:
|
|
3425
|
+
if out.get(node):
|
|
3426
|
+
truncated = True
|
|
3427
|
+
return
|
|
3428
|
+
for line, dst, via, subscriber in out.get(node, ()):
|
|
3429
|
+
if dst in seen or dst not in self.nodes:
|
|
3430
|
+
continue
|
|
3431
|
+
if via == "event" and subscriber not in seen:
|
|
3432
|
+
continue # nobody in this flow subscribed, so the handler does not run here
|
|
3433
|
+
if via in ("dispatch", "di") and home and self.file_lang.get(home) == "csharp" \
|
|
3434
|
+
and not self._dispatch_reaches(home, dst):
|
|
3435
|
+
continue # an implementation in a project this program does not reference
|
|
3436
|
+
if kind == "test" and via == "runs":
|
|
3437
|
+
continue
|
|
3438
|
+
if len(steps) >= max_steps:
|
|
3439
|
+
truncated = True
|
|
3440
|
+
return
|
|
3441
|
+
seen.add(dst)
|
|
3442
|
+
seq = len(steps)
|
|
3443
|
+
steps.append((seq, depth + 1, dst, via, int(line) if line < 10 ** 9 - 1 else None, parent_seq))
|
|
3444
|
+
# A launched program is its own composition: what it can reach is judged from there.
|
|
3445
|
+
walk(dst, depth + 1, seq, self.file_of.get(dst, home) if via == "process" else home)
|
|
3446
|
+
walk(start, 0, 0, entry_file)
|
|
3447
|
+
n = self.nodes[start]
|
|
3448
|
+
name = _test_title(n) if n.kind == "test" else start.split(":", 2)[-1].split("::")[-1]
|
|
3449
|
+
mods = {self.nodes[self.file_of[s[2]]].parent_id for s in steps if s[2] in self.file_of}
|
|
3450
|
+
self.flows.append((fid, name, "static", start, 0.0, None, "fact", SOURCE, {
|
|
3451
|
+
"kind": kind, "detail": detail, "steps": len(steps), "truncated": truncated,
|
|
3452
|
+
"modules": sorted(m for m in mods if m)}))
|
|
3453
|
+
self.flow_steps.add(fid, steps)
|
|
3454
|
+
|
|
3455
|
+
def _flow_select(self, out: dict, starts: list) -> Optional[set]:
|
|
3456
|
+
"""The starts to walk, given the call graph the walk reads (`out`); None for all of them.
|
|
3457
|
+
leyline.incremental walks again only the flows an edit can have changed."""
|
|
3458
|
+
return None
|
|
3459
|
+
|
|
3460
|
+
def _py_symbol(self, node_id: str, argc: int) -> Optional[list[Node]]:
|
|
3461
|
+
n = self.nodes.get(node_id)
|
|
3462
|
+
if n is None:
|
|
3463
|
+
return None
|
|
3464
|
+
if n.kind == "callable":
|
|
3465
|
+
return [n] if argc < 0 else (self._pick([n], argc) or [n])
|
|
3466
|
+
if n.kind == "type":
|
|
3467
|
+
init = next((found for c in CTORS for found in [self._methods(n.id, c, argc)] if found), None)
|
|
3468
|
+
self.edges.append(Edge("instantiates", self._last_src, n.id, "heuristic"))
|
|
3469
|
+
return init or None
|
|
3470
|
+
return None
|
|
3471
|
+
|
|
3472
|
+
def _py_bare(self, fid: str, call: CallSite, defined_here: bool) -> Optional[list[Node]]:
|
|
3473
|
+
self._last_src = call.src_id
|
|
3474
|
+
lang = self.file_lang[fid]
|
|
3475
|
+
own = f"{_repo_of(fid)}:{lang}:{self._modpath(fid)}.{call.name}"
|
|
3476
|
+
hit = self._py_symbol(own, call.argc)
|
|
3477
|
+
if hit is not None:
|
|
3478
|
+
return hit
|
|
3479
|
+
if self._py_bound(fid, call.src_id, call.name):
|
|
3480
|
+
target, symbol = self._py_name(fid, call.src_id, call.name)
|
|
3481
|
+
return self._py_symbol(self._py_export(target, symbol or call.name) or "", call.argc) if target else None
|
|
3482
|
+
if lang == "typescript" and own not in self.nodes and call.name not in self.outside_imports[fid]:
|
|
3483
|
+
# Not declared here and not imported by name: a local taken out of something (`const { save } = await
|
|
3484
|
+
# load()`). If exactly one exported function reachable from this file has the name, say so as a guess.
|
|
3485
|
+
reach = self._reach(fid)
|
|
3486
|
+
cands = [c for c in self.by_name.get((lang, call.name), ())
|
|
3487
|
+
if self.file_of.get(c.id) in reach and c.parent_id == self.file_of.get(c.id) and c.attrs.get("visibility") == "public"]
|
|
3488
|
+
if len(cands) == 1 and len(call.name) > 3:
|
|
3489
|
+
self._guessed = True
|
|
3490
|
+
self.stats[BY_LANGUAGE[lang].NAME]["calls_by_unique_name"] += 1
|
|
3491
|
+
return cands
|
|
3492
|
+
return None # builtins, star imports, and classes with no constructor
|
|
3493
|
+
|
|
3494
|
+
# -- write ---------------------------------------------------------------
|
|
3495
|
+
def _owner(self, node_id: str) -> str:
|
|
3496
|
+
"""The repository a node's rows are written under."""
|
|
3497
|
+
r = _repo_of(node_id)
|
|
3498
|
+
return r if r in self.repos else self.repo
|
|
3499
|
+
|
|
3500
|
+
def _final(self) -> tuple[list, list]:
|
|
3501
|
+
"""The edges and calls as they are written: edges whose ends both exist, each once, links between
|
|
3502
|
+
repositories marked, and a contains edge per parent; calls whose caller exists, each once, sorted.
|
|
3503
|
+
Made once per run."""
|
|
3504
|
+
got = getattr(self, "_final_rows", None)
|
|
3505
|
+
if got is not None:
|
|
3506
|
+
return got
|
|
3507
|
+
multi, owner = len(self.repos) > 1, self._owner
|
|
3508
|
+
# Drop edges whose endpoints did not survive (defensive) and exact duplicates.
|
|
3509
|
+
seen, edges = set(), []
|
|
3510
|
+
for e in self.edges:
|
|
3511
|
+
key = (e.kind, e.src_id, e.dst_id, tuple(sorted((e.attrs or {}).items(), key=str)).__repr__())
|
|
3512
|
+
if e.src_id in self.nodes and e.dst_id in self.nodes and key not in seen:
|
|
3513
|
+
seen.add(key)
|
|
3514
|
+
if multi and owner(e.dst_id) != owner(e.src_id):
|
|
3515
|
+
e.attrs = {**(e.attrs or {}), "to_repo": owner(e.dst_id)} # a link between repositories
|
|
3516
|
+
edges.append(e)
|
|
3517
|
+
for n in self.nodes.values():
|
|
3518
|
+
if n.parent_id:
|
|
3519
|
+
edges.append(Edge("contains", n.parent_id, n.id, "exact", None)) # no attrs: not an empty dict per node
|
|
3520
|
+
calls = sorted({c for c in self.calls if c[0] in self.nodes})
|
|
3521
|
+
self._final_rows = (edges, calls)
|
|
3522
|
+
return self._final_rows
|
|
3523
|
+
|
|
3524
|
+
def _write(self, con) -> None:
|
|
3525
|
+
owner = self._owner
|
|
3526
|
+
with con:
|
|
3527
|
+
for repo, root in self.repos.items():
|
|
3528
|
+
store.clear_facts(con, repo)
|
|
3529
|
+
store.write_nodes(con, [n for n in self.nodes.values() if owner(n.id) == repo], repo, SOURCE, self.commits[repo])
|
|
3530
|
+
store.set_root(con, repo, root)
|
|
3531
|
+
edges, calls = self._final()
|
|
3532
|
+
for repo in self.repos: # each row carries the commit of the repository its source is in
|
|
3533
|
+
store.write_edges(con, [e for e in edges if owner(e.src_id) == repo], SOURCE, self.commits[repo])
|
|
3534
|
+
store.write_calls(con, [c for c in calls if owner(c[0]) == repo], self.commits[repo])
|
|
3535
|
+
store.write_flows(con, self.repo, self.flows, self.flow_steps)
|
|
3536
|
+
self._write_coverage(con, len(self.flows), len(self.flow_steps))
|
|
3537
|
+
store.rebuild_derived(con)
|
|
3538
|
+
|
|
3539
|
+
def _write_coverage(self, con, flows: int, steps: int) -> None:
|
|
3540
|
+
"""What each extractor did, per repo, with the run's counts."""
|
|
3541
|
+
multi = len(self.repos) > 1
|
|
3542
|
+
from .adapters import ADAPTERS
|
|
3543
|
+
# Counts are for the whole run; in a workspace every member repo gets the same rows, marked as such.
|
|
3544
|
+
ws = {"workspace": sorted(self.repos)} if multi else {}
|
|
3545
|
+
for repo in self.repos:
|
|
3546
|
+
commit = self.commits[repo]
|
|
3547
|
+
for a in ADAPTERS:
|
|
3548
|
+
st = dict(self.stats.get(a.NAME, {}))
|
|
3549
|
+
status = "ok" if st.get("files") else "no_files"
|
|
3550
|
+
store.write_coverage(con, repo, a.NAME, a.VERSION, status, commit, {**st, **ws} if st else st)
|
|
3551
|
+
for channel in ("event", "process", "di", "http", "rpc", "queue", "db", "file", "format"):
|
|
3552
|
+
store.write_coverage(con, repo, f"communicates:{channel}", "0.1", "ok", commit,
|
|
3553
|
+
dict(self.channel_stats.get(channel, {})))
|
|
3554
|
+
store.write_coverage(con, repo, "flows:static", "0.1", "ok", commit, {"flows": flows, "steps": steps, **ws})
|
|
3555
|
+
for name in ("exact:roslyn", "exact:scip"):
|
|
3556
|
+
info = dict(self.exact_stats.get(name, {}))
|
|
3557
|
+
status = info.pop("status", "not_analyzed")
|
|
3558
|
+
store.write_coverage(con, repo, name, "1" if status == "ok" else "-",
|
|
3559
|
+
status if status in ("ok", "failed") else "not_analyzed", commit, info)
|
|
3560
|
+
kept = con.execute("SELECT format, stats FROM coverage_runs ORDER BY created DESC LIMIT 1").fetchone()
|
|
3561
|
+
if kept: # an imported coverage file outlives a re-index
|
|
3562
|
+
store.write_coverage(con, repo, "coverage", kept["format"], "ok", commit, json.loads(kept["stats"] or "{}"))
|
|
3563
|
+
else:
|
|
3564
|
+
store.write_coverage(con, repo, "coverage", "-", "not_analyzed", commit, {})
|
|
3565
|
+
if multi:
|
|
3566
|
+
con.execute("INSERT OR REPLACE INTO meta VALUES ('workspace', ?)", (json.dumps(list(self.repos)),))
|
|
3567
|
+
|
|
3568
|
+
|
|
3569
|
+
def _normalize(parts: tuple) -> list[str]:
|
|
3570
|
+
out: list[str] = []
|
|
3571
|
+
for p in parts:
|
|
3572
|
+
if p == "..":
|
|
3573
|
+
if out:
|
|
3574
|
+
out.pop()
|
|
3575
|
+
elif p not in (".", ""):
|
|
3576
|
+
out.append(p)
|
|
3577
|
+
return out
|
|
3578
|
+
|
|
3579
|
+
|
|
3580
|
+
def first_commit(root: Path) -> Optional[str]:
|
|
3581
|
+
"""The repository's first commit (the earliest of its root commits), which a moved or renamed clone keeps."""
|
|
3582
|
+
out = _git(root, "rev-list", "--max-parents=0", "HEAD", timeout=60)
|
|
3583
|
+
return sorted(out.split())[0] if out else None
|
|
3584
|
+
|
|
3585
|
+
|
|
3586
|
+
def _moved_members(con, given: list[tuple[Path, str]], stored: list[str]) -> list[tuple[Path, str]]:
|
|
3587
|
+
"""A directory named to map that is a stored member moved or renamed keeps that member's id, so its code is
|
|
3588
|
+
not added a second time. It is that member when the member's directory is gone and both are git repositories
|
|
3589
|
+
with the same first commit and, where both have an origin, the same origin. Anything less sure (no git, a
|
|
3590
|
+
store that never recorded the first commit) is a new member; `leyline map --forget <id>` drops the old one."""
|
|
3591
|
+
held = store.roots(con)
|
|
3592
|
+
names = {rid for _, rid in given}
|
|
3593
|
+
gone = [rid for rid in stored if rid not in names and not (held.get(rid) and Path(held[rid]).is_dir())]
|
|
3594
|
+
if not gone:
|
|
3595
|
+
return given
|
|
3596
|
+
out = []
|
|
3597
|
+
for root, rid in given:
|
|
3598
|
+
if rid not in stored and root not in {Path(p).resolve() for p in held.values()}:
|
|
3599
|
+
first = first_commit(root)
|
|
3600
|
+
url = _git(root, "remote", "get-url", "origin")
|
|
3601
|
+
for old in list(gone):
|
|
3602
|
+
was = con.execute("SELECT value FROM meta WHERE key = ?", (f"first_commit:{old}",)).fetchone()
|
|
3603
|
+
attrs = con.execute("SELECT attrs FROM nodes WHERE id = ? AND kind = 'repo'", (old,)).fetchone()
|
|
3604
|
+
old_url = (json.loads(attrs[0] or "{}") if attrs else {}).get("url")
|
|
3605
|
+
if first and was and was[0] == first and (not url or not old_url or url == old_url):
|
|
3606
|
+
print(f"leyline: {root.name} is workspace member {old}, moved from {held.get(old)}; mapped as {old}",
|
|
3607
|
+
file=sys.stderr)
|
|
3608
|
+
rid = old
|
|
3609
|
+
gone.remove(old)
|
|
3610
|
+
break
|
|
3611
|
+
out.append((root, rid))
|
|
3612
|
+
return out
|
|
3613
|
+
|
|
3614
|
+
|
|
3615
|
+
def workspace(con, roots: list[str | Path], repo_id: Optional[str] = None) -> list[tuple[Path, str]]:
|
|
3616
|
+
"""The (root, repo id) pairs to index together. Repositories named together make the store a workspace,
|
|
3617
|
+
and it remembers them: indexing any one of them later indexes all of them again, so the links between
|
|
3618
|
+
them are kept. A member whose directory is gone is left as it was stored."""
|
|
3619
|
+
given = [(Path(r).resolve(), repo_id if repo_id and len(roots) == 1 else Path(r).resolve().name) for r in roots]
|
|
3620
|
+
row = con.execute("SELECT value FROM meta WHERE key = 'workspace'").fetchone()
|
|
3621
|
+
stored = json.loads(row[0]) if row else []
|
|
3622
|
+
if len(given) == 1 and not stored:
|
|
3623
|
+
return given
|
|
3624
|
+
given = _moved_members(con, given, stored)
|
|
3625
|
+
out = list(given)
|
|
3626
|
+
for rid in stored:
|
|
3627
|
+
here = store.roots(con).get(rid)
|
|
3628
|
+
root = (str(here),) if here is not None else None
|
|
3629
|
+
if any(rid == g[1] or (root and Path(root[0]) == g[0]) for g in given):
|
|
3630
|
+
continue
|
|
3631
|
+
if root and Path(root[0]).is_dir():
|
|
3632
|
+
out.append((Path(root[0]), rid))
|
|
3633
|
+
else:
|
|
3634
|
+
print(f"leyline: workspace member {rid} is not at {root[0] if root else '?'}; its facts are kept as they were",
|
|
3635
|
+
file=sys.stderr)
|
|
3636
|
+
ids = [rid for _, rid in out]
|
|
3637
|
+
dupes = sorted({i for i in ids if ids.count(i) > 1})
|
|
3638
|
+
if dupes:
|
|
3639
|
+
raise ValueError(f"two repositories would have the id {dupes[0]!r}; index them from directories with different names")
|
|
3640
|
+
return out
|
|
3641
|
+
|
|
3642
|
+
|
|
3643
|
+
def index(root: str | Path | list, db_path: str | Path, repo_id: Optional[str] = None, exact: str = "off",
|
|
3644
|
+
scip: Optional[list[str]] = None, full: bool = False, _verify: bool = True,
|
|
3645
|
+
listed: Optional[dict] = None) -> dict:
|
|
3646
|
+
"""Index a repository, or several as one workspace (`root` a list). `exact` is off, auto, roslyn or scip:
|
|
3647
|
+
whether a compiler's view of the references replaces the syntax-based one (see leyline.exact). `scip`
|
|
3648
|
+
lists index.scip files. When the store was made by an earlier run, only what changed since is done again
|
|
3649
|
+
(leyline.incremental), and the store comes out as a full run would leave it; `full` makes it a full run."""
|
|
3650
|
+
from . import incremental, tours
|
|
3651
|
+
|
|
3652
|
+
con = store.connect(db_path)
|
|
3653
|
+
inc = None
|
|
3654
|
+
try:
|
|
3655
|
+
members = workspace(con, list(root) if isinstance(root, (list, tuple)) else [root], repo_id)
|
|
3656
|
+
ix = Indexer(members[0][0], members[0][1], members[1:])
|
|
3657
|
+
ix.listed = dict(listed or {}) # repo id -> its files or its Listing, when the caller has listed them
|
|
3658
|
+
ix.exact_mode, ix.scip_paths = ("scip" if scip and exact == "off" else exact), list(scip or [])
|
|
3659
|
+
ix.keep_results = False
|
|
3660
|
+
inc = incremental.Run(con, db_path, ix, full=full)
|
|
3661
|
+
began = time.perf_counter()
|
|
3662
|
+
# The run makes millions of objects that live until it ends; the cycle collector would scan them over and
|
|
3663
|
+
# over (a third of loading a large repository's cached parse output) and find nothing to free.
|
|
3664
|
+
gc.disable()
|
|
3665
|
+
try:
|
|
3666
|
+
stats = ix.run(con)
|
|
3667
|
+
finally:
|
|
3668
|
+
gc.enable()
|
|
3669
|
+
stats.update(ix.exact_stats)
|
|
3670
|
+
left = {r: skipped_summary(ix.skipped.get(r, []), ix.failed.get(r, [])) for r in ix.repos}
|
|
3671
|
+
stats["left_out"] = left if len(ix.repos) > 1 else left[ix.repo]
|
|
3672
|
+
with con: # so the overview and the map page can say what is not in the map
|
|
3673
|
+
for r, v in left.items():
|
|
3674
|
+
con.execute("INSERT OR REPLACE INTO meta VALUES (?, ?)", (f"left_out:{r}", json.dumps(v)))
|
|
3675
|
+
for r, r_root in ix.repos.items(): # who each member is, to know it again if its folder moves
|
|
3676
|
+
if con.execute("SELECT 1 FROM meta WHERE key = ?", (f"first_commit:{r}",)).fetchone() is None:
|
|
3677
|
+
first = first_commit(r_root)
|
|
3678
|
+
if first:
|
|
3679
|
+
con.execute("INSERT INTO meta VALUES (?, ?)", (f"first_commit:{r}", first))
|
|
3680
|
+
# Everything from here reads the store. Letting the indexer go first keeps its memory (most of a GB on a
|
|
3681
|
+
# large repo) from adding to what clustering and the pattern matchers hold.
|
|
3682
|
+
repo, timing, repos = ix.repo, ix.timing, dict(ix.repos)
|
|
3683
|
+
inc.release()
|
|
3684
|
+
del ix
|
|
3685
|
+
mark = time.perf_counter()
|
|
3686
|
+
multi = len(repos) > 1
|
|
3687
|
+
systems = {r: inc.systems(con, r) for r in repos}
|
|
3688
|
+
stats["systems"] = systems if multi else systems[repo]
|
|
3689
|
+
if inc.full:
|
|
3690
|
+
with con: # a workspace re-clusters every repo, so it rebuilds everything
|
|
3691
|
+
store.rebuild_derived(con, systems_of=None if multi else repo)
|
|
3692
|
+
timing["systems"] = round(time.perf_counter() - mark, 3)
|
|
3693
|
+
mark = time.perf_counter()
|
|
3694
|
+
stats["patterns"] = inc.patterns(con, repo)
|
|
3695
|
+
for other in list(repos)[1:]:
|
|
3696
|
+
with con:
|
|
3697
|
+
store.write_coverage(con, other, "patterns:structural", "0.1", "ok", None, stats["patterns"])
|
|
3698
|
+
tour = {r: tours.generate(con, r) for r in repos}
|
|
3699
|
+
stats["tour"] = tour if multi else tour[repo]
|
|
3700
|
+
timing["patterns_and_tour"] = round(time.perf_counter() - mark, 3)
|
|
3701
|
+
stats["stale_annotations"] = store.refresh_stale(con)
|
|
3702
|
+
if multi:
|
|
3703
|
+
stats["workspace"] = {"repos": {r: str(p) for r, p in repos.items()}, "cross_repo": _cross_counts(con)}
|
|
3704
|
+
files = sum(v.get("files", 0) for k, v in stats.items() if k.startswith(("tree-sitter", "generic")))
|
|
3705
|
+
lines = con.execute(f"SELECT COALESCE(SUM(span_end), 0) FROM nodes WHERE kind = 'file' AND repo_id IN ({','.join('?' * len(repos))})",
|
|
3706
|
+
list(repos)).fetchone()[0]
|
|
3707
|
+
total = round(time.perf_counter() - began, 3)
|
|
3708
|
+
stats["timing"] = {"total_seconds": total, **timing, "files": files, "lines": lines,
|
|
3709
|
+
"lines_per_second": round(lines / total) if total else 0}
|
|
3710
|
+
stats["incremental"] = inc.summary()
|
|
3711
|
+
with con: # kept in the store so a later reader can see what the map cost to build
|
|
3712
|
+
for r in repos:
|
|
3713
|
+
con.execute("INSERT OR REPLACE INTO meta VALUES (?, ?)", (f"timing:{r}", json.dumps(stats["timing"])))
|
|
3714
|
+
con.execute("INSERT OR REPLACE INTO extractor_coverage VALUES (?,?,?,?,?,?)",
|
|
3715
|
+
(r, "timing", "-", "ok", None, json.dumps(stats["timing"])))
|
|
3716
|
+
# Which Leyline made this map, noted once the whole of it is written: a newer one, or a run that stopped
|
|
3717
|
+
# part way (its systems, patterns or tour not yet redone), maps again even where no file changed
|
|
3718
|
+
# (see loop.refresh).
|
|
3719
|
+
con.execute("INSERT OR REPLACE INTO meta VALUES ('made_by', ?)", (incremental.code_version(),))
|
|
3720
|
+
inc.finish(con)
|
|
3721
|
+
if _verify and stats["incremental"]["mode"] == "incremental" and os.environ.get("LEYLINE_VERIFY"):
|
|
3722
|
+
stats["incremental"]["differs"] = incremental.verify(db_path, members, exact, scip, listed)
|
|
3723
|
+
inc = None
|
|
3724
|
+
return stats
|
|
3725
|
+
finally:
|
|
3726
|
+
if inc is not None:
|
|
3727
|
+
inc.abandon()
|
|
3728
|
+
con.close()
|
|
3729
|
+
|
|
3730
|
+
|
|
3731
|
+
def _cross_counts(con) -> list[dict]:
|
|
3732
|
+
from .query import cross_repo
|
|
3733
|
+
return cross_repo(con)["pairs"]
|