leyline-code 0.8.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. leyline/__init__.py +3 -0
  2. leyline/adapters/__init__.py +19 -0
  3. leyline/adapters/csharp.py +1026 -0
  4. leyline/adapters/generic.py +844 -0
  5. leyline/adapters/python.py +929 -0
  6. leyline/adapters/typescript.py +1512 -0
  7. leyline/affected.py +509 -0
  8. leyline/agent_skills.py +311 -0
  9. leyline/change.py +500 -0
  10. leyline/channels.py +1782 -0
  11. leyline/cli.py +1029 -0
  12. leyline/cluster.py +149 -0
  13. leyline/context.py +531 -0
  14. leyline/coupling.py +485 -0
  15. leyline/coverage.py +376 -0
  16. leyline/diagrams.py +680 -0
  17. leyline/diff.py +1078 -0
  18. leyline/drift.py +747 -0
  19. leyline/exact.py +410 -0
  20. leyline/explain.py +973 -0
  21. leyline/export.py +673 -0
  22. leyline/fncoupling.py +328 -0
  23. leyline/formats.py +923 -0
  24. leyline/grade.py +207 -0
  25. leyline/incremental.py +931 -0
  26. leyline/indexer.py +3733 -0
  27. leyline/learnings.py +743 -0
  28. leyline/loop.py +584 -0
  29. leyline/model.py +142 -0
  30. leyline/outline.py +1066 -0
  31. leyline/packing.py +86 -0
  32. leyline/patterns.py +478 -0
  33. leyline/pr.py +1249 -0
  34. leyline/props.py +255 -0
  35. leyline/query.py +683 -0
  36. leyline/quick.py +853 -0
  37. leyline/related.py +266 -0
  38. leyline/removal.py +231 -0
  39. leyline/renames.py +231 -0
  40. leyline/rereview.py +375 -0
  41. leyline/roslyn_refs/Program.cs +179 -0
  42. leyline/roslyn_refs/RoslynRefs.csproj +16 -0
  43. leyline/rules.py +171 -0
  44. leyline/schema.sql +178 -0
  45. leyline/scip_pb2.py +97 -0
  46. leyline/server.py +1230 -0
  47. leyline/skills/leyline-adversarial-review/SKILL.md +121 -0
  48. leyline/skills/leyline-ask/SKILL.md +57 -0
  49. leyline/skills/leyline-change-impact/SKILL.md +79 -0
  50. leyline/skills/leyline-explain-flow/SKILL.md +70 -0
  51. leyline/skills/leyline-explore-module/SKILL.md +55 -0
  52. leyline/skills/leyline-pr-review/SKILL.md +77 -0
  53. leyline/skills/leyline-quick-change/SKILL.md +63 -0
  54. leyline/skills/leyline-spec/SKILL.md +179 -0
  55. leyline/skills/leyline-tour/SKILL.md +46 -0
  56. leyline/spec.py +1819 -0
  57. leyline/store.py +560 -0
  58. leyline/tours.py +352 -0
  59. leyline/verdicts.py +390 -0
  60. leyline/viewer/viewer.html +1622 -0
  61. leyline_code-0.8.1.dist-info/METADATA +1325 -0
  62. leyline_code-0.8.1.dist-info/RECORD +65 -0
  63. leyline_code-0.8.1.dist-info/WHEEL +4 -0
  64. leyline_code-0.8.1.dist-info/entry_points.txt +2 -0
  65. leyline_code-0.8.1.dist-info/licenses/LICENSE +674 -0
leyline/indexer.py ADDED
@@ -0,0 +1,3733 @@
1
+ """Walk a repo, run the language adapters, resolve names across files, and write facts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import gc
6
+ import hashlib
7
+ import itertools
8
+ import posixpath
9
+ import json
10
+ import os
11
+ import pickle
12
+ import re
13
+ import subprocess
14
+ import time
15
+ import sys
16
+ from array import array
17
+ from collections import Counter, defaultdict
18
+ from pathlib import Path
19
+ from typing import Optional
20
+
21
+ from . import packing, store
22
+ from .adapters import BY_EXTENSION, BY_LANGUAGE
23
+ from .adapters.python import module_path
24
+ from .model import CallSite, Edge, Endpoint, FieldUse, FileResult, Node
25
+
26
+ SOURCE = "leyline-indexer/0.1"
27
+ MODULE_MARKERS = ("pyproject.toml", "setup.py", "package.json", "__init__.py")
28
+ # Languages where a file is a module and other files name what they take from it (import { a } from "./b").
29
+ FILE_MODULE = ("python", "typescript")
30
+ CTORS = (".ctor", "__init__", "constructor")
31
+ NODE_BUILTINS = frozenset("""assert async_hooks buffer child_process cluster console constants crypto dgram dns domain
32
+ events fs http http2 https inspector module net os path perf_hooks process punycode querystring readline repl stream
33
+ string_decoder timers tls tty url util v8 vm worker_threads zlib test""".split())
34
+ SKIP_DIRS = {".git", "node_modules", "bin", "obj", "__pycache__", ".venv", "venv", ".godot", ".leyline"}
35
+
36
+
37
+ # Directories of other people's code, left out even when committed: mapped, they bury the repository's own code.
38
+ DEPENDENCY_DIRS = {"node_modules", "bower_components", "site-packages", "__pycache__"}
39
+ # A vendor directory is only someone else's code when a package manager says so (Go modules, Composer).
40
+ VENDOR_MARKERS = ("vendor/modules.txt", "vendor/autoload.php")
41
+ MAX_FILE_MB = 5 # a source file larger than this is generated; parsing one costs seconds and hundreds of MB
42
+ LONG_LINES = 2000 # average characters per line above which a file is minified or generated, not written
43
+
44
+
45
+ def _git(root: Path, *args: str, raw: bool = False, timeout: float = 30):
46
+ try:
47
+ out = subprocess.run(["git", "-C", str(root), *args], capture_output=True, timeout=timeout,
48
+ stdin=subprocess.DEVNULL)
49
+ except (OSError, subprocess.SubprocessError):
50
+ return None
51
+ if out.returncode != 0:
52
+ return None
53
+ # git writes paths and names as UTF-8 bytes whatever the platform's locale is
54
+ return out.stdout if raw else out.stdout.decode("utf-8", "replace").strip()
55
+
56
+
57
+ def max_file_bytes() -> int:
58
+ env = os.environ.get("LEYLINE_MAX_FILE_MB", "")
59
+ try:
60
+ return int(float(env) * 2**20) if env else MAX_FILE_MB * 2**20
61
+ except ValueError:
62
+ return MAX_FILE_MB * 2**20
63
+
64
+
65
+ class Listing:
66
+ """The files of a repository that can be read, and the ones left out with the reason for each. `how` is git or
67
+ walk; `why_walk` says why git did not list them, when it is a git repository."""
68
+
69
+ def __init__(self, files: list[str], skipped: list[tuple[str, str]], how: str, why_walk: Optional[str] = None):
70
+ self.files, self.skipped, self.how, self.why_walk = files, skipped, how, why_walk
71
+
72
+
73
+ def scan(root: Path, names: Optional[list[str]] = None) -> Listing:
74
+ """Every file to index: what git lists (tracked, and untracked but not ignored), or a walk of the directory when
75
+ it is not a git repository. Anything that cannot be read as a source file is left out, with a reason, so that
76
+ nothing later fails on it: entries git lists that are not files on disk (submodules, deleted files, symlinks to
77
+ directories or to nothing), files outside the repository, other people's code, and source files that are too
78
+ large, binary, minified or unreadable. `names` lists the files instead, as git would (a commit's files written
79
+ out by `leyline pr`)."""
80
+ root = Path(root)
81
+ skipped: list[tuple[str, str]] = []
82
+ why_walk = None
83
+ out = None if names is not None else \
84
+ _git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard", raw=True, timeout=600)
85
+ if names is not None:
86
+ how, given, names = "git", names, []
87
+ for n in given:
88
+ try:
89
+ n.encode("utf-8")
90
+ names.append(n)
91
+ except UnicodeEncodeError:
92
+ skipped.append((n.encode("utf-8", "surrogateescape").decode("utf-8", "replace"), "file name is not UTF-8"))
93
+ names = sorted(set(names))
94
+ elif out is not None:
95
+ names, how = [], "git"
96
+ for b in out.split(b"\0"):
97
+ if not b:
98
+ continue
99
+ try:
100
+ names.append(b.decode("utf-8"))
101
+ except UnicodeDecodeError:
102
+ skipped.append((b.decode("utf-8", "replace"), "file name is not UTF-8"))
103
+ names = sorted(set(n.rstrip("/") for n in names)) # an untracked nested repository is listed as "dir/"
104
+ else:
105
+ names, how = _walk(root, skipped), "walk"
106
+ if (root / ".git").exists():
107
+ try:
108
+ run = subprocess.run(["git", "-C", str(root), "ls-files", "-z"], capture_output=True, timeout=600,
109
+ stdin=subprocess.DEVNULL)
110
+ why_walk = run.stderr.decode("utf-8", "replace").strip().splitlines()
111
+ why_walk = why_walk[0] if why_walk else f"git exited with {run.returncode}"
112
+ except (OSError, subprocess.SubprocessError) as e:
113
+ why_walk = f"git could not be run ({e})"
114
+ vendored = {m[: -len(m.rsplit("/", 1)[-1])] for m in names
115
+ if any(m == v or m.endswith("/" + v) for v in VENDOR_MARKERS)}
116
+ try:
117
+ real_root = root.resolve()
118
+ except OSError:
119
+ real_root = root
120
+ limit = max_file_bytes()
121
+ files, listed = [], set(names)
122
+ for f in names:
123
+ parts = f.split("/")
124
+ if set(parts[:-1]) & DEPENDENCY_DIRS or any(f.startswith(v) for v in vendored):
125
+ skipped.append((f, "dependency directory"))
126
+ continue
127
+ why = _unusable(root, real_root, f, BY_EXTENSION, limit, listed)
128
+ if why:
129
+ skipped.append((f, why))
130
+ else:
131
+ files.append(f)
132
+ return Listing(files, skipped, how, why_walk)
133
+
134
+
135
+ def _walk(root: Path, skipped: list) -> list[str]:
136
+ """The files under root when git cannot list them. Symlinked directories are not followed (a link back up the
137
+ tree would never end), nested repositories and build or tool directories are left out, and the ignore files git
138
+ would apply are: the person's global excludes file, .git/info/exclude, then each .gitignore (see _ignore_rules).
139
+ git may fail on a real repository (a broken .git, a dubious-ownership refusal), and a file kept out of it with
140
+ one of those (.env) must stay out of the map and its page here too."""
141
+ files = []
142
+ top = [r for f in (_global_excludes(), root / ".git" / "info" / "exclude") if f is not None
143
+ for r in _ignore_rules(f, "")]
144
+
145
+ def err(e: OSError) -> None:
146
+ try:
147
+ rel = Path(e.filename).relative_to(root).as_posix() if e.filename else "?"
148
+ except ValueError:
149
+ rel = str(e.filename)
150
+ skipped.append((rel, f"cannot read directory: {e.strerror or e}"))
151
+ rules: dict[str, list] = {} # directory -> the ignore rules in force there, its own last
152
+ for d, dirs, names in os.walk(root, onerror=err):
153
+ rel = Path(d).relative_to(root).as_posix()
154
+ rel = "" if rel == "." else rel + "/"
155
+ here = rules.get(rel[:-1].rpartition("/")[0] + "/" if "/" in rel[:-1] else "", []) if rel else top
156
+ if ".gitignore" in names:
157
+ here = here + _ignore_rules(Path(d) / ".gitignore", rel)
158
+ rules[rel] = here
159
+ keep = []
160
+ for x in dirs:
161
+ p = os.path.join(d, x)
162
+ if x in SKIP_DIRS or x in DEPENDENCY_DIRS or _ignored(here, rel + x, True):
163
+ continue
164
+ if os.path.islink(p):
165
+ skipped.append((rel + x, "symlink to a directory"))
166
+ elif os.path.exists(os.path.join(p, ".git")):
167
+ skipped.append((rel + x, "nested repository (map it on its own, or with this one as a workspace)"))
168
+ else:
169
+ keep.append(x)
170
+ dirs[:] = sorted(keep)
171
+ for x in names:
172
+ try:
173
+ (rel + x).encode("utf-8")
174
+ except UnicodeEncodeError:
175
+ skipped.append(((rel + x).encode("utf-8", "surrogateescape").decode("utf-8", "replace"),
176
+ "file name is not UTF-8"))
177
+ continue
178
+ if not _ignored(here, rel + x, False):
179
+ files.append(rel + x)
180
+ return sorted(files)
181
+
182
+
183
+ def _global_excludes() -> Optional[Path]:
184
+ """The person's global ignore file: git's core.excludesFile, else $XDG_CONFIG_HOME/git/ignore (~/.config/git/
185
+ ignore). Asked of git outside any repository, so a repository's own config has no say."""
186
+ try:
187
+ run = subprocess.run(["git", "config", "--get", "core.excludesFile"], capture_output=True, timeout=30,
188
+ stdin=subprocess.DEVNULL, cwd=str(Path.home()))
189
+ named = run.stdout.decode("utf-8", "replace").strip() if run.returncode == 0 else ""
190
+ except (OSError, subprocess.SubprocessError, RuntimeError):
191
+ named = ""
192
+ if named:
193
+ return Path(os.path.expanduser(named))
194
+ xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
195
+ return Path(xdg) / "git" / "ignore"
196
+
197
+
198
+ def _ignore_rules(path: Path, base: str) -> list[tuple]:
199
+ """An ignore file's patterns as (regex, negated, directories only), read as git reads them: names, globs with
200
+ * ? [..] and **, a leading / or an inner / to anchor, a trailing / for directories, ! to take a path back, a
201
+ backslash before a character to mean it as written (\\#, \\!, \\*, a trailing \\ ), and trailing spaces cut."""
202
+ import re as _re
203
+ try:
204
+ lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
205
+ except OSError:
206
+ return []
207
+ out = []
208
+ for line in lines:
209
+ while line.endswith(" ") and not line.endswith("\\ "): # trailing spaces go; git keeps an escaped one
210
+ line = line[:-1]
211
+ if not line or line.startswith("#"):
212
+ continue
213
+ neg = line.startswith("!")
214
+ if neg:
215
+ line = line[1:]
216
+ dir_only = line.endswith("/")
217
+ line = line.rstrip("/") if dir_only else line
218
+ anchored = "/" in line # a leading or inner slash; the trailing one only says "a directory"
219
+ line = line.lstrip("/")
220
+ if not line:
221
+ continue
222
+ rx, i = "", 0
223
+ while i < len(line):
224
+ c = line[i]
225
+ if c == "\\" and i + 1 < len(line): # the next character as written
226
+ rx, i = rx + _re.escape(line[i + 1]), i + 2
227
+ elif line.startswith("**/", i):
228
+ rx, i = rx + "(?:.*/)?", i + 3
229
+ elif line.startswith("/**", i) and i + 3 == len(line):
230
+ rx, i = rx + "/.*", i + 3
231
+ elif line.startswith("**", i):
232
+ rx, i = rx + ".*", i + 2
233
+ elif c == "*":
234
+ rx, i = rx + "[^/]*", i + 1
235
+ elif c == "?":
236
+ rx, i = rx + "[^/]", i + 1
237
+ elif c == "[" and "]" in line[i + 1:]:
238
+ j = line.index("]", i + 1)
239
+ body = line[i + 1:j]
240
+ rx, i = rx + "[" + ("^" + body[1:] if body.startswith("!") else body) + "]", j + 1
241
+ else:
242
+ rx, i = rx + _re.escape(c), i + 1
243
+ prefix = _re.escape(base) if anchored else _re.escape(base) + "(?:.*/)?"
244
+ out.append((_re.compile(prefix + rx + "$"), neg, dir_only))
245
+ return out
246
+
247
+
248
+ def _ignored(rules: list, path: str, is_dir: bool) -> bool:
249
+ hit = False
250
+ for rx, neg, dir_only in rules: # the last rule that matches decides, as in git
251
+ if (is_dir or not dir_only) and rx.match(path):
252
+ hit = not neg
253
+ return hit
254
+
255
+
256
+ def _unusable(root: Path, real_root: Path, f: str, sources: dict, limit: int, listed: set) -> Optional[str]:
257
+ """Why a listed path cannot be indexed, or None. Only source files (those an adapter reads) are opened."""
258
+ import errno
259
+ import stat as st
260
+ p = root / f
261
+ try:
262
+ info = os.lstat(p)
263
+ except FileNotFoundError:
264
+ return "deleted (git still lists it)"
265
+ except OSError as e:
266
+ return f"cannot read: {e.strerror or e}"
267
+ except ValueError: # UnicodeEncodeError: a name the file system's encoding (the locale's) cannot spell
268
+ return "file name cannot be spelled in this system's encoding (use a UTF-8 locale)"
269
+ if st.S_ISLNK(info.st_mode):
270
+ try:
271
+ target = p.resolve(strict=True)
272
+ info = os.stat(p)
273
+ except FileNotFoundError:
274
+ return "broken symlink"
275
+ except RuntimeError: # a symlink loop, on Python before 3.13
276
+ return "symlink loop"
277
+ except OSError as e:
278
+ return "symlink loop" if e.errno == errno.ELOOP else f"cannot read: {e.strerror or e}"
279
+ if st.S_ISDIR(info.st_mode):
280
+ return "symlink to a directory"
281
+ try:
282
+ inside = target.relative_to(real_root).as_posix()
283
+ except ValueError:
284
+ return "symlink to outside the repository"
285
+ if inside in listed:
286
+ return "symlink to another listed file (indexed there)"
287
+ if st.S_ISDIR(info.st_mode):
288
+ return "submodule or nested repository (map it on its own, or with this one as a workspace)"
289
+ if not st.S_ISREG(info.st_mode):
290
+ return "not a regular file"
291
+ ext = "." + f.rsplit(".", 1)[-1] if "." in f.rsplit("/", 1)[-1] else ""
292
+ if ext not in sources:
293
+ return None
294
+ if info.st_size > limit:
295
+ return f"larger than {limit / 2**20:g} MB (generated?)"
296
+ if f.endswith((".min.js", ".min.mjs", ".min.cjs")):
297
+ return "minified"
298
+ try:
299
+ with open(p, "rb") as fh:
300
+ head = fh.read(65536)
301
+ except OSError as e:
302
+ return f"cannot read: {e.strerror or e}"
303
+ if _binary(head[:8192]):
304
+ return "binary"
305
+ # a bundle of a few tens of KB on one line is as minified as a larger one; a short file is let through
306
+ if len(head) >= 4 * LONG_LINES and head.count(b"\n") < len(head) // LONG_LINES:
307
+ return "minified or generated (very long lines)"
308
+ return None
309
+
310
+
311
+ _CONTROL = bytes(b for b in range(32) if b not in (9, 10, 12, 13, 27)) + b"\x7f"
312
+
313
+
314
+ def _binary(head: bytes) -> bool:
315
+ """Whether the start of a file is data rather than text. A NUL byte alone does not decide it (source can hold
316
+ one inside a string literal); one byte in twenty being a control character does. UTF-16 text is half NULs, so a
317
+ byte order mark lets it through."""
318
+ if not head or head.startswith((b"\xff\xfe", b"\xfe\xff")):
319
+ return False
320
+ return len(head) - len(head.translate(None, _CONTROL)) > len(head) // 20
321
+
322
+
323
+ def list_files(root: Path) -> list[str]:
324
+ return scan(root).files
325
+
326
+
327
+ def _report_skipped(repo: str, skipped: list) -> None:
328
+ """One line per reason on stderr, with a few of the files, so a person can see what the map leaves out."""
329
+ quiet = ("dependency directory", "deleted (git still lists it)", "symlink to another listed file (indexed there)")
330
+ by = defaultdict(list)
331
+ for f, why in skipped:
332
+ by[why].append(f)
333
+ for why, fs in sorted(by.items(), key=lambda kv: -len(kv[1])):
334
+ if why in quiet:
335
+ line = f"leyline: {repo}: left out {len(fs)} file{'s' * (len(fs) != 1)}: {why}"
336
+ else:
337
+ shown = ", ".join(fs[:3]) + (f" and {len(fs) - 3} more" if len(fs) > 3 else "")
338
+ line = f"leyline: {repo}: left out {why}: {shown}"
339
+ if REPORTED is None or line not in REPORTED: # a command that maps twice (a pull request's base and head)
340
+ if REPORTED is not None: # says it once
341
+ REPORTED.add(line)
342
+ print(line, file=sys.stderr)
343
+
344
+
345
+ REPORTED: Optional[set] = None # the lines said so far by the command running (leyline.cli sets it)
346
+
347
+
348
+ def skipped_summary(skipped: list, failed: list) -> dict:
349
+ """What a run left out and why, for the stats and the store: reason -> count and a few examples."""
350
+ out: dict = {}
351
+ for f, why in list(skipped) + [(f, "failed to parse: " + (w or "")) for f, w in failed]:
352
+ key = why if not why.startswith("failed to parse") else "failed to parse"
353
+ got = out.setdefault(key, {"count": 0, "examples": []})
354
+ got["count"] += 1
355
+ if len(got["examples"]) < 5:
356
+ got["examples"].append(f if key != "failed to parse" else f"{f}: {why[len('failed to parse: '):]}")
357
+ return out
358
+
359
+
360
+ # The placeholders a table-driven test's title is filled in from (it.each, test.each, pytest ids): printf-style
361
+ # %s %d %j ..., $name and ${name} from a table, and Jest's %# for the row number.
362
+ _TITLE_SLOT = re.compile(r"%[sdifjoOpc#]|\$\{?[A-Za-z_][\w.]*\}?")
363
+
364
+
365
+ def _test_title(n) -> str:
366
+ """A test's name as a flow shows it. A title that is a template (`%s/%s` from it.each) says little on its own,
367
+ so it is shown after its suite, with each placeholder as an ellipsis."""
368
+ title = n.name
369
+ if not _TITLE_SLOT.search(title):
370
+ return title
371
+ shown = _TITLE_SLOT.sub("…", title.replace("%%", "%")).strip()
372
+ suite = _TITLE_SLOT.sub("…", str(n.attrs.get("suite") or "")).strip()
373
+ return f"{suite} > {shown}" if suite else shown
374
+
375
+
376
+ def read_source(path: Path) -> bytes:
377
+ """A source file's bytes as the parsers take them: UTF-8. A UTF-16 file (it starts with a byte order mark,
378
+ as some Windows editors write C#) is converted; lines stay where they were."""
379
+ data = Path(path).read_bytes()
380
+ if data.startswith((b"\xff\xfe", b"\xfe\xff")):
381
+ try:
382
+ return data.decode("utf-16").encode("utf-8")
383
+ except UnicodeDecodeError:
384
+ return data
385
+ if str(path).endswith((".py", ".pyi", ".pyw")): # a PEP 263 declaration in the first two lines: # coding: latin-1
386
+ m = _CODING.match(b"\n".join(data.split(b"\n", 2)[:2]))
387
+ if m and codecs_lookup(m[1].decode("ascii")) not in (None, "utf-8"):
388
+ try:
389
+ return data.decode(m[1].decode("ascii")).encode("utf-8")
390
+ except (UnicodeDecodeError, LookupError):
391
+ return data
392
+ return data
393
+
394
+
395
+ def _jsonc(path: Path) -> Optional[dict]:
396
+ """A JSON file that may hold comments and trailing commas, as tsconfig.json does; None when it cannot be read."""
397
+ try:
398
+ text = path.read_text(encoding="utf-8-sig", errors="replace")
399
+ except OSError:
400
+ return None
401
+ # comments outside strings go, then commas before a closing bracket
402
+ text = re.sub(r'("(?:\\.|[^"\\])*")|//[^\n]*|/\*.*?\*/', lambda m: m[1] or "", text, flags=re.S)
403
+ text = re.sub(r'("(?:\\.|[^"\\])*")|,(\s*[}\]])', lambda m: m[1] or m[2], text)
404
+ try:
405
+ data = json.loads(text)
406
+ except (ValueError, RecursionError): # nested past the recursion limit: Python before 3.13 raises this
407
+ return None
408
+ return data if isinstance(data, dict) else None
409
+
410
+
411
+ _CODING = re.compile(rb"(?:[ \t\f]*(?:#[^\n]*)?\n)?[ \t\f]*#[^\n]*?coding[:=][ \t]*([-\w.]+)")
412
+
413
+
414
+ def codecs_lookup(name: str) -> Optional[str]:
415
+ """A codec's canonical name, or None when Python has no such codec."""
416
+ import codecs
417
+ try:
418
+ return codecs.lookup(name).name
419
+ except LookupError:
420
+ return None
421
+
422
+
423
+ def source_lines(path: Path) -> list[str]:
424
+ """A source file's lines, numbered as the parsers number them: only a newline ends a line (str.splitlines also
425
+ breaks at form feeds and other separators, which put every later line out of step with the syntax tree)."""
426
+ return text_lines(read_source(path))
427
+
428
+
429
+ def text_lines(data: bytes) -> list[str]:
430
+ return [ln[:-1] if ln.endswith("\r") else ln for ln in data.decode("utf-8", "replace").split("\n")]
431
+
432
+
433
+ def _module_dirs(files: list[str]) -> set[str]:
434
+ """Directories that are modules because they hold a project marker."""
435
+ dirs, packages = set(), set()
436
+ for f in files:
437
+ d, _, base = f.rpartition("/")
438
+ if base.endswith(".csproj") or base in MODULE_MARKERS:
439
+ dirs.add(d)
440
+ if base == "__init__.py":
441
+ packages.add(d)
442
+ # A nested __init__.py belongs to its top-most package, not to a module of its own. A project file
443
+ # inside another project's folder (a workspace package) is a module of its own. Both are looked up by
444
+ # directory: scanning every file per package, and every directory per directory, was quadratic.
445
+ projects = {f.rpartition("/")[0] for f in files
446
+ if f.endswith(".csproj") or f.rsplit("/", 1)[-1] in MODULE_MARKERS[:3]}
447
+ own = {d for d in dirs if d not in packages or d in projects}
448
+
449
+ def nested(d: str) -> bool:
450
+ parts = d.split("/")
451
+ return any("/".join(parts[:k]) in dirs for k in range(1, len(parts)))
452
+ return {d for d in dirs if d in own or not nested(d)}
453
+
454
+
455
+ def _module_for(path: str, module_dirs: set[str]) -> str:
456
+ d = path.rpartition("/")[0]
457
+ cur = d
458
+ while True:
459
+ if cur in module_dirs and cur != "":
460
+ return cur
461
+ if "/" not in cur:
462
+ break
463
+ cur = cur.rpartition("/")[0]
464
+ if cur in module_dirs and cur != "":
465
+ return cur
466
+ return d # no marker: the file's own directory ("" is the repo root)
467
+
468
+
469
+ def _ext_clashes(work: list) -> set[str]:
470
+ """The files whose ids keep their extension: files of one language in one directory that differ only in their
471
+ extension (a.js beside a.ts, m.c beside m.h) would name their contents alike. The first by path keeps the usual
472
+ ids, so a directory with no such pair keeps them all; the others keep the extension in theirs."""
473
+ import inspect
474
+ groups: dict[tuple, list] = defaultdict(list)
475
+ for w in work:
476
+ adapter = BY_EXTENSION[w[1]]
477
+ if "keep_ext" in inspect.signature(adapter.parse).parameters:
478
+ groups[(adapter.LANGUAGE, adapter.module_path(w[0]))].append(w[0])
479
+ return {f for fs in groups.values() if len(fs) > 1 for f in sorted(fs)[1:]}
480
+
481
+
482
+ def _arity(type_id: str) -> int:
483
+ tail = type_id.rsplit("`", 1)
484
+ return int(tail[1]) if len(tail) == 2 and tail[1].isdigit() else 0
485
+
486
+
487
+ NOT_TYPES = ("String", "Self", "None", "Some", "Ok", "Err")
488
+ _IDENT = re.compile(r"[A-Za-z_]\w*")
489
+
490
+
491
+ def _declared(text: list[str], start: Optional[int], end: Optional[int]) -> dict:
492
+ """Variables a span of text declares with a type: name -> the type names written for it, in the order that
493
+ decides (the first, then any later one), so that which of them are in the repo can be settled later."""
494
+ body = "\n".join(text[(start or 1) - 1:end or start or 1])
495
+ out: dict[str, list] = {}
496
+ for pat, gn, gt in Indexer._DECL:
497
+ for m in pat.finditer(body):
498
+ name, tname = m.group(gn), m.group(gt)
499
+ seen = out.setdefault(name, [])
500
+ if tname not in seen:
501
+ seen.append(tname)
502
+ return out
503
+
504
+
505
+ def _parse_one(job):
506
+ """Parse one file: (path, ext, module dir, module id, lines, sha1, pickled (result, declarations) or None,
507
+ error or None, the names the result mentions). Runs in a worker process on large repositories, so it touches
508
+ nothing but its arguments. The result comes back pickled because that is how it would cross from the worker
509
+ anyway, and the same bytes are what the parse cache keeps (leyline.incremental); its calls are packed apart
510
+ (leyline.packing)."""
511
+ from .incremental import mentions
512
+ try:
513
+ out = _parse_file(job)
514
+ res, decls = out[6], out[8]
515
+ if res is None:
516
+ return (*out[:6], None, out[7], b"")
517
+ return (*out[:6], packing.pack(res, decls), None, mentions(res, decls))
518
+ except Exception as exc: # MemoryError, or a result that will not pickle: the file fails, not the run
519
+ f, ext, mod_dir, mod_id = job[2]
520
+ return f, ext, mod_dir, mod_id, 1, None, None, f"{type(exc).__name__}: {exc}"[:300], b""
521
+
522
+
523
+ DEEP_RECURSION = 25_000 # frames allowed when a file is parsed again on a large stack; past this, the work grows too fast
524
+
525
+
526
+ def _deep(fn, *args):
527
+ """fn(*args) on a thread with a 1 GB stack and a recursion limit to match. The adapters walk the syntax tree
528
+ recursively; a file nested deeper than the default limit allows (a long else-if chain, a generated expression)
529
+ is read this way instead of being lost. The stack is only reserved address space until it is used."""
530
+ import threading
531
+ out: list = []
532
+
533
+ def run():
534
+ try:
535
+ out.append((True, fn(*args)))
536
+ except BaseException as exc: # handed back to the caller's thread
537
+ out.append((False, exc))
538
+ if os.name == "nt" and sys.version_info < (3, 12):
539
+ # Python before 3.12 on Windows does not stop a deep recursion before the thread's stack runs out: the whole
540
+ # process dies. The file is reported as too deeply nested instead.
541
+ raise RecursionError("nested too deeply to read")
542
+ old_stack, old_limit = threading.stack_size(), sys.getrecursionlimit()
543
+ try:
544
+ threading.stack_size(2**30)
545
+ except (ValueError, RuntimeError): # a platform that will not give a thread a large stack
546
+ raise RecursionError("nested too deeply to read")
547
+ try:
548
+ sys.setrecursionlimit(max(old_limit, DEEP_RECURSION))
549
+ t = threading.Thread(target=run, name="leyline-deep-parse")
550
+ t.start()
551
+ t.join()
552
+ finally:
553
+ threading.stack_size(old_stack)
554
+ sys.setrecursionlimit(old_limit)
555
+ ok, value = out[0]
556
+ if not ok:
557
+ raise value
558
+ return value
559
+
560
+
561
+ def _parse_file(job):
562
+ root, repo, (f, ext, mod_dir, mod_id) = job[:3]
563
+ try:
564
+ data = read_source(Path(root) / f)
565
+ except OSError as exc: # gone or made unreadable since it was listed
566
+ return f, ext, mod_dir, mod_id, 1, None, None, f"cannot read: {exc.strerror or exc}", None
567
+ loc, sha = data.count(b"\n") + 1, hashlib.sha1(data).hexdigest()
568
+ adapter = BY_EXTENSION[ext]
569
+ args = (repo, f, f"{repo}:file:{f}", data, mod_dir or ".")
570
+ parse = adapter.parse
571
+ if len(job) > 3 and job[3]: # see _ext_clashes
572
+ import functools
573
+ parse = functools.partial(adapter.parse, keep_ext=True)
574
+ try:
575
+ try:
576
+ res = parse(*args)
577
+ except RecursionError:
578
+ res = _deep(parse, *args)
579
+ except RecursionError:
580
+ return f, ext, mod_dir, mod_id, loc, sha, None, "nested too deeply to read", None
581
+ except Exception as exc:
582
+ return f, ext, mod_dir, mod_id, loc, sha, None, f"{type(exc).__name__}: {exc}"[:300], None
583
+ lines = data.split(b"\n")
584
+ for n in res.nodes:
585
+ if n.span_start and n.kind in ("type", "callable", "test", "field"):
586
+ # A hash of the node's own text, so a later index can tell which nodes were edited.
587
+ n.content_hash = hashlib.sha1(b"\n".join(ln.strip() for ln in lines[n.span_start - 1:n.span_end])).hexdigest()[:16]
588
+ decls = None
589
+ if getattr(adapter, "GENERIC", False):
590
+ # The generic resolver reads variable types out of function and type text. Reading it here puts that
591
+ # work in the parallel part; the text is read the way Indexer._file_text reads it, so the answer is the same.
592
+ text = text_lines(data)
593
+ if text is not None:
594
+ # Only where a call names a variable receiver is the type looked for; any other place falls back to
595
+ # reading the text in the main process.
596
+ holders, todo = set(), list(res.calls)
597
+ while todo:
598
+ c = todo.pop()
599
+ if c.receiver not in (None, "this", "base", "?") and _IDENT.fullmatch(c.receiver):
600
+ holders.update((c.src_id, c.enclosing_type))
601
+ if c.chain is not None:
602
+ todo.append(c.chain)
603
+ decls = {n.id: (n.span_start, n.span_end, _declared(text, n.span_start, n.span_end))
604
+ for n in res.nodes if n.id in holders}
605
+ return f, ext, mod_dir, mod_id, loc, sha, res, None, decls
606
+
607
+
608
+ PARALLEL_MIN_FILES = 300 # below this, starting worker processes costs more than it saves
609
+
610
+
611
+ def _jobs() -> int:
612
+ env = os.environ.get("LEYLINE_JOBS")
613
+ if env and env.isdigit():
614
+ return max(1, int(env))
615
+ return max(1, min(os.cpu_count() or 1, 16))
616
+
617
+
618
+ def _start_method() -> Optional[str]:
619
+ """How worker processes start. fork is quickest, but it is only safe in a process with one thread (a fork
620
+ taken while another thread holds a lock deadlocks the child), and macOS and Windows do not have it safely or
621
+ at all; spawn works everywhere. LEYLINE_START_METHOD picks one (fork, forkserver or spawn)."""
622
+ import multiprocessing
623
+ import threading
624
+ env = os.environ.get("LEYLINE_START_METHOD")
625
+ methods = multiprocessing.get_all_start_methods()
626
+ if env:
627
+ if env not in methods:
628
+ raise ValueError(f"LEYLINE_START_METHOD={env!r}: this platform has {', '.join(methods)}")
629
+ return env
630
+ if sys.platform.startswith("linux") and "fork" in methods and threading.active_count() == 1:
631
+ return "fork"
632
+ return "spawn"
633
+
634
+
635
+ CHUNK = 16 # files a worker takes at a time; small, so that few files are in flight when a worker dies
636
+
637
+
638
+ def _timeout() -> float:
639
+ """Seconds a worker may spend on one batch of files (or, retried alone, one file) before it is taken to be
640
+ stuck and stopped. LEYLINE_PARSE_TIMEOUT changes it."""
641
+ try:
642
+ return max(1.0, float(os.environ.get("LEYLINE_PARSE_TIMEOUT", "") or 600))
643
+ except ValueError:
644
+ return 600.0
645
+
646
+
647
+ def _parse_all(root: Path, repo: str, work: list, died: Optional[list] = None, keep_ext: frozenset = frozenset()):
648
+ """Parse every file, across processes when there are enough files to pay for starting them.
649
+ Results come back in the order given, so the index is the same however many processes ran.
650
+ A worker that dies (a crash inside a parser, or the system killing it for memory) or gets stuck loses only the
651
+ file it was on: the files that were in flight are parsed again one at a time, each in a fresh process, and the
652
+ one that kills or stalls its process is reported as failed (and named in `died`)."""
653
+ jobs = _jobs()
654
+ items = [(str(root), repo, w, w[0] in keep_ext) for w in work]
655
+ if jobs == 1 or len(items) < PARALLEL_MIN_FILES:
656
+ yield from map(_parse_one, items)
657
+ return
658
+ import multiprocessing
659
+ from collections import deque
660
+ from concurrent.futures import ProcessPoolExecutor, TimeoutError as Stuck
661
+ from concurrent.futures.process import BrokenProcessPool
662
+ ctx = multiprocessing.get_context(_start_method())
663
+ limit = _timeout()
664
+
665
+ def pool(n):
666
+ return ProcessPoolExecutor(max_workers=n, mp_context=ctx, initializer=_worker_init)
667
+
668
+ chunks = [items[i:i + CHUNK] for i in range(0, len(items), CHUNK)]
669
+ pos = 0
670
+ worked = {"any": False} # whether a worker has ever returned a result in this run
671
+ while pos < len(chunks):
672
+ ahead: deque = deque()
673
+ nxt = pos
674
+ ex = pool(jobs)
675
+ try:
676
+ while pos < len(chunks):
677
+ # A few chunks per worker are handed out ahead of the one awaited: enough to keep every worker
678
+ # busy, few enough that when one dies the file that killed it is among them.
679
+ while nxt < len(chunks) and len(ahead) < 4 * jobs:
680
+ ahead.append(ex.submit(_parse_chunk, chunks[nxt]))
681
+ nxt += 1
682
+ try:
683
+ done = ahead[0].result(timeout=limit)
684
+ except Stuck:
685
+ _kill(ex)
686
+ raise BrokenProcessPool("stuck") from None
687
+ ahead.popleft()
688
+ pos += 1
689
+ worked["any"] = True
690
+ yield from done
691
+ except BrokenProcessPool:
692
+ suspects = [it for c in chunks[pos:nxt] for it in c]
693
+ ex.shutdown(wait=True)
694
+ yield from _parse_alone(suspects, pool, died, limit, worked)
695
+ if worked.get("broken"): # workers cannot run here at all: the rest is parsed in this process
696
+ yield from map(_parse_one, [it for c in chunks[nxt:] for it in c])
697
+ return
698
+ pos = nxt
699
+ finally:
700
+ ex.shutdown(wait=True)
701
+
702
+
703
+ def _worker_init() -> None:
704
+ # Ctrl+C reaches every process in the group; the main process stops the run, and the workers stay quiet
705
+ # instead of each printing a traceback.
706
+ import signal
707
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
708
+
709
+
710
+ def _kill(ex) -> None:
711
+ for proc in list(getattr(ex, "_processes", {}).values()): # the pool has no public way to stop a busy worker
712
+ try:
713
+ proc.kill()
714
+ except Exception:
715
+ pass
716
+
717
+
718
+ def _parse_chunk(items):
719
+ return [_parse_one(it) for it in items]
720
+
721
+
722
+ def _parse_alone(items, pool, died, limit, worked):
723
+ """Parse each file in a one-worker pool, starting a new pool after a file kills or stalls one."""
724
+ from concurrent.futures import TimeoutError as Stuck
725
+ from concurrent.futures.process import BrokenProcessPool
726
+
727
+ def failed(it, why):
728
+ f, ext, mod_dir, mod_id = it[2]
729
+ print(f"leyline: {why}: {f}; it is left out", file=sys.stderr)
730
+ if died is not None:
731
+ died.append(f)
732
+ try:
733
+ data = read_source(Path(it[0]) / f)
734
+ loc, sha = data.count(b"\n") + 1, hashlib.sha1(data).hexdigest()
735
+ except OSError:
736
+ loc, sha = 1, None
737
+ return f, ext, mod_dir, mod_id, loc, sha, None, why, b""
738
+ ex = pool(1)
739
+ # Files whose worker died before any worker in this run had finished a file: until one does, it is not known
740
+ # whether the files kill the workers or the workers cannot run here at all.
741
+ lost: list = []
742
+ try:
743
+ for k, it in enumerate(items):
744
+ if len(lost) >= 3:
745
+ print("leyline: worker processes are not working here; parsing in this process instead",
746
+ file=sys.stderr)
747
+ worked["broken"] = True
748
+ yield from map(_parse_one, [x for x, _ in lost] + list(items[k:]))
749
+ return
750
+ why = None
751
+ try:
752
+ got = ex.submit(_parse_one, it).result(timeout=limit)
753
+ except Stuck:
754
+ why = f"parsing took longer than {limit:g} s"
755
+ _kill(ex)
756
+ except BrokenProcessPool:
757
+ why = "the parser process died on this file"
758
+ if why is not None:
759
+ ex.shutdown(wait=True)
760
+ ex = pool(1)
761
+ if not worked["any"]:
762
+ lost.append((it, why))
763
+ continue
764
+ got = failed(it, why)
765
+ else:
766
+ worked["any"] = True
767
+ for x, w in lost: # a worker can run, so those files did kill theirs
768
+ yield failed(x, w)
769
+ lost = []
770
+ yield got
771
+ for x, w in lost:
772
+ yield failed(x, w)
773
+ finally:
774
+ ex.shutdown(wait=True)
775
+
776
+
777
+ class FlowSteps:
778
+ """The steps of every flow, packed into arrays: as tuples they were the largest thing in memory on a large
779
+ repository (millions of them). Iterates as (flow id, seq, depth, callable id, via, site line, parent seq)."""
780
+
781
+ def __init__(self):
782
+ self.flows: list[tuple] = [] # (flow id, index of its first step)
783
+ self.callable: list[str] = [] # the node ids themselves, shared with the nodes
784
+ # depth is at most the walk's max_depth, a parent is a step of the same flow, and a line is under 10**9
785
+ self.depth, self.via, self.line, self.parent = array("h"), array("b"), array("i"), array("i")
786
+ self.via_names: list[str] = []
787
+ self._via: dict[str, int] = {}
788
+
789
+ def add(self, flow_id: str, steps: list[tuple]) -> None:
790
+ """steps: (seq, depth, callable id, via, site line or None, parent seq or None), seq counting from 0."""
791
+ self.flows.append((flow_id, len(self.callable)))
792
+ for _seq, depth, c, via, line, parent in steps:
793
+ v = self._via.get(via)
794
+ if v is None:
795
+ v = self._via[via] = len(self.via_names)
796
+ self.via_names.append(via)
797
+ self.callable.append(c)
798
+ self.depth.append(depth)
799
+ self.via.append(v)
800
+ self.line.append(-1 if line is None else line)
801
+ self.parent.append(-1 if parent is None else parent)
802
+
803
+ def __len__(self) -> int:
804
+ return len(self.callable)
805
+
806
+ def ids(self):
807
+ """Every flow id, then every callable id, in step order: the ids the rows name, as store._keys takes them."""
808
+ return itertools.chain((fid for fid, _ in self.flows), self.callable)
809
+
810
+ def __iter__(self):
811
+ ends = [start for _, start in self.flows[1:]] + [len(self.callable)]
812
+ for (fid, start), end in zip(self.flows, ends):
813
+ for i in range(start, end):
814
+ line, parent = self.line[i], self.parent[i]
815
+ yield (fid, i - start, self.depth[i], self.callable[i], self.via_names[self.via[i]],
816
+ None if line < 0 else line, None if parent < 0 else parent)
817
+
818
+
819
+ def flow_ids(starts: list) -> list:
820
+ """(start, kind, detail) -> (start, kind, detail, flow id), each start walked once. A flow is `flow:<start>`;
821
+ a start that is also another kind of start (an entry point that is a test too) gets `flow:<start>#<kind>` for
822
+ the later ones, so neither flow replaces the other."""
823
+ out, taken = [], set()
824
+ for s in dict.fromkeys(starts):
825
+ start, kind, _ = s
826
+ fid, n = f"flow:{start}", 2
827
+ if fid in taken:
828
+ fid = f"flow:{start}#{kind}"
829
+ while fid in taken:
830
+ fid, n = f"flow:{start}#{kind}{n}", n + 1
831
+ taken.add(fid)
832
+ out.append((*s, fid))
833
+ return out
834
+
835
+
836
+ def _repo_of(node_id: str) -> str:
837
+ """Every node id starts with its repo id: flask:python:..., flask:file:..., flask:module:..."""
838
+ return node_id.split(":", 1)[0]
839
+
840
+
841
+ _WILDCARD = re.compile(r"\*\w*|\(\.\*\)|:\w+(\*|\+|\(\.\*\))|\{\*\*?\w*\}")
842
+
843
+
844
+ def _wildcard(segment: str) -> bool:
845
+ """A route segment that takes the rest of the path: Fastify's and Express's `*`, Express's `(.*)`, `:path*` and
846
+ `*name`, ASP.NET's `{*path}`."""
847
+ return bool(_WILDCARD.fullmatch(segment))
848
+
849
+
850
+ def _segment_fits(route: str, request: str) -> bool:
851
+ """One segment of a route against one of a request: equal, or a parameter of the route (ASP.NET's [controller]
852
+ is the class's name, which clients write in lower case)."""
853
+ return route == request or route.startswith(("<", "{", ":")) or route.lower() == request.lower()
854
+
855
+
856
+ def _fits_wildcard(route: list, request: list) -> bool:
857
+ """Does a request's path fit a route that ends in a wildcard? The wildcard takes one segment or more; a hole in
858
+ the request (`/api/lore/${rel}`, written {}) may itself be several segments, so it can cover the route's
859
+ parameters up to and including the wildcard. A hole never stands in for a segment the route spells out."""
860
+ k = len(route) - 1
861
+
862
+ def go(i: int, j: int) -> bool:
863
+ if i == k:
864
+ return j < len(request)
865
+ if j == len(request) or not _segment_fits(route[i], request[j]):
866
+ return False
867
+ if go(i + 1, j + 1):
868
+ return True
869
+ if request[j] != "{}":
870
+ return False
871
+ i2 = i + 1
872
+ while i2 < k and route[i2].startswith(("<", "{", ":")):
873
+ if go(i2 + 1, j + 1):
874
+ return True
875
+ i2 += 1
876
+ return i2 == k and j == len(request) - 1
877
+ return go(0, 0)
878
+
879
+
880
+ class Indexer:
881
+ def __init__(self, root: str | Path, repo_id: Optional[str] = None,
882
+ others: Optional[list[tuple[str | Path, str]]] = None):
883
+ """`others` are more (root, repo id) pairs indexed in the same run, a workspace: a name in one
884
+ repository resolves to its declaration in another. Node ids keep their own repo's prefix."""
885
+ self.root = Path(root).resolve()
886
+ self.repo = repo_id or self.root.name
887
+ self.repos: dict[str, Path] = {self.repo: self.root}
888
+ for r, rid in others or ():
889
+ if rid in self.repos or ":" in rid:
890
+ raise ValueError(f"repo id {rid!r} is used twice or contains ':'")
891
+ self.repos[rid] = Path(r).resolve()
892
+ self.commit = _git(self.root, "rev-parse", "HEAD")
893
+ self.commits = {rid: _git(r, "rev-parse", "HEAD") for rid, r in self.repos.items()}
894
+ self.nodes: dict[str, Node] = {}
895
+ self.edges: list[Edge] = []
896
+ self.calls: list[tuple] = []
897
+ self.results: dict[str, FileResult] = {} # file id -> adapter output
898
+ # On a large repository, file id -> its calls and field uses, packed (leyline.packing); see _open
899
+ self._packed: dict[str, bytes] = {}
900
+ self._packed_decls: dict[str, bytes] = {}
901
+ self._opened: Optional[list] = None # [file id, result, changed] of the one file unpacked now
902
+ self._ckeys: Optional[dict] = None
903
+ self._file_index: dict[str, int] = {}
904
+ self.file_lang: dict[str, str] = {}
905
+ self.file_of_path: dict[str, str] = {}
906
+ self.keep_ext: set[str] = set() # file ids whose node ids keep the file's extension (_ext_clashes)
907
+ self._decl_cache: dict[str, dict] = {}
908
+ self._read_decls: dict[str, dict] = {} # file id -> node id -> (span, declarations read in the parse)
909
+ self._text_cache: dict[str, list] = {}
910
+ self.stats: dict[str, Counter] = defaultdict(Counter)
911
+ self.flows: list[tuple] = []
912
+ self.call_col: dict[tuple, int] = {}
913
+ self.flow_steps = FlowSteps()
914
+ self.channel_stats: dict[str, Counter] = defaultdict(Counter)
915
+ self.project_refs: dict[str, set] = defaultdict(set)
916
+ self._vis_cache: dict[str, set] = {}
917
+ self._sets: dict[tuple, set] = {} # see _file_set
918
+ self._name_ix: dict[tuple, dict] = {}
919
+ self._loose_reach: dict[str, set] = {}
920
+ self.exact_mode = "off" # off | auto | roslyn | scip
921
+ self.scip_paths: list[str] = []
922
+ self.listed: dict = {} # repo -> its Listing, or its files, when the caller has listed them (see scan)
923
+ self.exact_stats: dict[str, dict] = {}
924
+ self.keep_results = True # False: run() lets go of the adapters' output and the resolvers' caches before writing
925
+ self.parse_cache = None # leyline.incremental: parse output kept from the last run, by content hash
926
+
927
+ def _apply_exact(self) -> None:
928
+ """Let a compiler overrule the syntax resolvers where one is available."""
929
+ if self.exact_mode == "off":
930
+ return
931
+ from . import exact
932
+ multi = len(self.repos) > 1
933
+ if self.exact_mode in ("auto", "roslyn") and any(v == "csharp" for v in self.file_lang.values()):
934
+ if multi:
935
+ records, info = [], {"status": "skipped", "reason": "not yet run for a workspace of several repositories"}
936
+ else:
937
+ try:
938
+ records, info = exact.roslyn(self)
939
+ except RuntimeError as exc:
940
+ records, info = [], {"status": "failed", "reason": str(exc)[-600:]}
941
+ if records:
942
+ info.update(exact.apply(self, records, "roslyn"))
943
+ self.exact_stats["exact:roslyn"] = info
944
+ paths = [(p, None) for p in self.scip_paths]
945
+ if self.exact_mode in ("auto", "scip") and not paths:
946
+ paths = [(str(p), repo) for repo, root in self.repos.items()
947
+ for p in (root / "index.scip", root / ".leyline" / "index.scip") if p.is_file()]
948
+ by_repo = {}
949
+ for path, repo in paths:
950
+ try:
951
+ if repo is None: # a SCIP index names the directory it was made in; that says whose it is
952
+ repo = next((r for r, root in self.repos.items() if exact.scip_root(path) == root), self.repo) if multi else self.repo
953
+ info = {"status": "ok", "file": path, **exact.apply(self, exact.scip(path, self.repos[repo]), "scip",
954
+ repo if multi else None)}
955
+ except Exception as exc:
956
+ info = {"status": "failed", "reason": str(exc)[-600:]}
957
+ self.exact_stats["exact:scip"] = info
958
+ by_repo[repo] = info
959
+ if multi and by_repo:
960
+ ok = any(i["status"] == "ok" for i in by_repo.values())
961
+ self.exact_stats["exact:scip"] = {"status": "ok" if ok else "failed", "by_repo": by_repo}
962
+
963
+ # -- public --------------------------------------------------------------
964
+ def _timed(self, name: str, fn, *args) -> None:
965
+ start = time.perf_counter()
966
+ fn(*args)
967
+ self.timing[name] = round(self.timing.get(name, 0) + time.perf_counter() - start, 3)
968
+
969
+ def run(self, con) -> dict:
970
+ self.timing: dict[str, float] = {}
971
+ started = time.perf_counter()
972
+ self.files_of, self.skipped, self.failed = {}, {}, {}
973
+ for rid, root in self.repos.items():
974
+ got = self.listed.get(rid) # listed already by the caller: its files, or the whole listing
975
+ listing = got if isinstance(got, Listing) else scan(root, got)
976
+ self.files_of[rid], self.skipped[rid] = listing.files, listing.skipped
977
+ if listing.why_walk:
978
+ print(f"leyline: git would not list the files of {root} ({listing.why_walk}); reading the directory"
979
+ " instead, with the common .gitignore patterns applied", file=sys.stderr)
980
+ elif listing.how == "walk":
981
+ print(f"leyline: {root} is not a git repository: reading every file under it, with the common"
982
+ " .gitignore patterns applied", file=sys.stderr)
983
+ _report_skipped(rid, listing.skipped)
984
+ for repo, root in self.repos.items():
985
+ self._parse_repo(repo, root, self.files_of[repo])
986
+ self.timing["parse"] = round(time.perf_counter() - started, 3)
987
+
988
+ def resolve():
989
+ for repo, files in self.files_of.items():
990
+ self._projects(files, repo)
991
+ self._attach_methods()
992
+ self._build_indexes()
993
+ self._resolve_imports()
994
+ self._script_entries()
995
+ self._resolve_types()
996
+ self._resolve_overrides()
997
+ self._resolve_calls()
998
+ self._resolve_events()
999
+ self._resolve_fields()
1000
+ self._timed("resolve", resolve)
1001
+ self._timed("compiler", self._apply_exact)
1002
+
1003
+ def channels():
1004
+ self._resolve_spawns()
1005
+ self._resolve_endpoints()
1006
+ from . import channels as more
1007
+ more.resolve(self) # dependency injection, queues, databases, RPC
1008
+ self._timed("channels", channels)
1009
+ if not self.keep_results:
1010
+ # Flows and the write are left, which read nodes, edges, calls (with their columns) and flows. What the
1011
+ # parse and the resolvers held (GBs on a large repository) is let go first, so the flows' lists and the
1012
+ # write's reuse that memory instead of adding to it.
1013
+ for held in (self.results, self._read_decls, self._decl_cache, self._name_ix, self._sets, self._text_cache,
1014
+ getattr(self, "_chain_memo", {}), self._packed, self._packed_decls,
1015
+ getattr(self, "_decls_unpacked", {})):
1016
+ held.clear()
1017
+ self._timed("flows", self._build_flows)
1018
+ if not self.keep_results:
1019
+ for held in (self._vis_cache, self.call_col, self._loose_reach):
1020
+ held.clear()
1021
+ self._timed("write", self._write, con)
1022
+ return {k: dict(v) for k, v in self.stats.items()}
1023
+
1024
+ def _parse_repo(self, repo: str, root: Path, files: list[str]) -> None:
1025
+ module_dirs = _module_dirs(files)
1026
+ self._add(Node(id=repo, kind="repo", name=repo, path="",
1027
+ attrs={"url": _git(root, "remote", "get-url", "origin"),
1028
+ "branch": _git(root, "rev-parse", "--abbrev-ref", "HEAD")}))
1029
+ work = []
1030
+ for f in files:
1031
+ ext = "." + f.rsplit(".", 1)[-1] if "." in f else ""
1032
+ adapter = BY_EXTENSION.get(ext)
1033
+ if adapter is None:
1034
+ continue
1035
+ mod_dir = _module_for(f, module_dirs)
1036
+ mod_id = self._module(mod_dir, files, repo)
1037
+ work.append((f, ext, mod_dir, mod_id))
1038
+ # With a parse cache (leyline.incremental), a file whose content and module are as they were last time is
1039
+ # not parsed again. Either way the files are taken in the listed order, so the index is the same.
1040
+ cache = self.parse_cache
1041
+ keep_ext = _ext_clashes(work)
1042
+ self.keep_ext.update(f"{repo}:file:{f}" for f in keep_ext)
1043
+ cached = cache.lookup(repo, root, work, keep_ext) if cache is not None else {}
1044
+ fresh = _parse_all(root, repo, [w for w in work if w[0] not in cached], keep_ext=keep_ext)
1045
+ failed_here = self.failed.setdefault(repo, [])
1046
+ packs = packing.packs(len(work))
1047
+ for w in work:
1048
+ got = cached.pop(w[0], None) # let go as it is used: on a large repository the blobs add up
1049
+ fresh_one = got is None
1050
+ if fresh_one:
1051
+ got = next(fresh)
1052
+ f, ext, mod_dir, mod_id, loc, sha, blob, failed, toks = got
1053
+ if cache is not None:
1054
+ cache.keep(f"{repo}:file:{f}", got, fresh_one, f in keep_ext)
1055
+ del got
1056
+ res, heavy, decls = packing.unpack(blob) if blob is not None else (None, None, None)
1057
+ blob = None
1058
+ adapter = BY_EXTENSION[ext]
1059
+ file_id = f"{repo}:file:{f}"
1060
+ self._add(Node(id=file_id, kind="file", name=f.rsplit("/", 1)[-1], parent_id=mod_id,
1061
+ language=adapter.LANGUAGE, path=f, span_start=1, span_end=loc,
1062
+ content_hash=sha, attrs={"loc": loc}))
1063
+ if res is None: # one bad file must not sink the run
1064
+ self.stats[adapter.NAME]["files_failed"] += 1
1065
+ failed_here.append((f, failed))
1066
+ print(f"leyline: failed to parse {f}: {failed}", file=sys.stderr)
1067
+ continue
1068
+ self.results[file_id] = res
1069
+ # Python's calls are kept unpacked: the argument-flow rule holds on to them after their file's turn.
1070
+ if packs and adapter.LANGUAGE != "python":
1071
+ self._packed[file_id] = heavy
1072
+ if decls is not None:
1073
+ self._packed_decls[file_id] = decls
1074
+ else:
1075
+ res.calls, res.field_uses = packing.unpack_heavy(heavy)
1076
+ if decls is not None:
1077
+ self._read_decls[file_id] = packing.unpack_decls(decls)
1078
+ self.file_of_path[f] = file_id
1079
+ self.file_lang[file_id] = adapter.LANGUAGE
1080
+ self.stats[adapter.NAME]["files"] += 1
1081
+ for n in res.nodes:
1082
+ self._add(n)
1083
+ if res.calls is not None:
1084
+ self._share_ids(res)
1085
+ self.edges.extend(res.edges)
1086
+ next(fresh, None) # lets the worker pool, if one was started, shut down
1087
+
1088
+ # -- packed calls (leyline.packing) -----------------------------------------
1089
+ def _open(self, fid: str, res: FileResult) -> FileResult:
1090
+ """The file's result with its calls and field uses in place. Packed ones are unpacked here, one file at a
1091
+ time: opening a file lets go of the one opened before (packing it again if it was changed)."""
1092
+ if res.calls is not None:
1093
+ return res
1094
+ self._shut()
1095
+ res.calls, res.field_uses = packing.unpack_heavy(self._packed[fid])
1096
+ self._share_ids(res)
1097
+ self._opened = [fid, res, False]
1098
+ return res
1099
+
1100
+ def _share_ids(self, res: FileResult) -> None:
1101
+ """Point each call's and field use's caller at its node's own id string. Unpacked apart from the nodes,
1102
+ each was a copy of it, and every call row made from the call kept that copy."""
1103
+ nodes = self.nodes
1104
+ for group in (res.calls, res.field_uses):
1105
+ for c in group:
1106
+ n = nodes.get(c.src_id)
1107
+ if n is not None:
1108
+ c.src_id = n.id
1109
+
1110
+ def _shut(self) -> None:
1111
+ if self._opened is None:
1112
+ return
1113
+ fid, res, changed = self._opened
1114
+ if changed:
1115
+ self._packed[fid] = packing.pack_heavy(res.calls, res.field_uses)
1116
+ res.calls = res.field_uses = None
1117
+ self._opened = self._ckeys = None
1118
+
1119
+ def _ckey(self, call: CallSite):
1120
+ """What _chain_memo keeps a call's results under: the call itself while it lives, which for a packed file
1121
+ is only while the file is open, so there a key that names the same call each time it is unpacked."""
1122
+ if self._opened is None:
1123
+ return id(call)
1124
+ if self._ckeys is None:
1125
+ fid = self._opened[0]
1126
+ self._ckeys = packing.call_keys(self._opened[1], self._file_index.setdefault(fid, len(self._file_index)))
1127
+ return self._ckeys.get(id(call), id(call))
1128
+
1129
+ def _decls_read(self, fid: Optional[str]) -> dict:
1130
+ """node id -> (span, declarations) the parse read in a file, unpacked for the last file asked about."""
1131
+ got = self._read_decls.get(fid)
1132
+ if got is None and fid in self._packed_decls:
1133
+ cache = self.__dict__.setdefault("_decls_unpacked", {})
1134
+ got = cache.get(fid)
1135
+ if got is None:
1136
+ if len(cache) >= 2:
1137
+ cache.clear()
1138
+ got = cache[fid] = packing.unpack_decls(self._packed_decls[fid])
1139
+ return got or {}
1140
+
1141
+ # -- structure -----------------------------------------------------------
1142
+ def _add(self, n: Node) -> None:
1143
+ if n.id in self.nodes:
1144
+ # Partial classes: keep the first declaration, note the extra file.
1145
+ first = self.nodes[n.id]
1146
+ if n.kind == "type" and n.path != first.path:
1147
+ first.attrs.setdefault("also_in", []).append(n.path)
1148
+ elif n.path == first.path and n.span_start and first.span_start and n.kind not in ("file", "module", "repo"):
1149
+ # One name defined twice in a file (if/else or try/except, a property's getter and setter): which one
1150
+ # runs is not known from the text, so it is one node over both, listing each, its text hash reading
1151
+ # every body.
1152
+ spans = first.attrs.setdefault("definitions", [[first.span_start, first.span_end or first.span_start]])
1153
+ spans.append([n.span_start, n.span_end or n.span_start])
1154
+ first.span_start = min(first.span_start, n.span_start)
1155
+ first.span_end = max(first.span_end or first.span_start, n.span_end or n.span_start)
1156
+ if first.content_hash and n.content_hash:
1157
+ first.content_hash = hashlib.sha1(f"{first.content_hash}|{n.content_hash}".encode()).hexdigest()[:16]
1158
+ return
1159
+ self.nodes[n.id] = n
1160
+
1161
+ def _module(self, mod_dir: str, files: list[str], repo: Optional[str] = None) -> str:
1162
+ repo = repo or self.repo
1163
+ mid = f"{repo}:module:{mod_dir or '.'}"
1164
+ if mid not in self.nodes:
1165
+ marker = self._markers(files).get(mod_dir)
1166
+ self._add(Node(id=mid, kind="module", name=mod_dir.rsplit("/", 1)[-1] or repo,
1167
+ parent_id=repo, path=mod_dir, attrs={"marker": marker}))
1168
+ return mid
1169
+
1170
+ def _markers(self, files: list[str]) -> dict:
1171
+ """directory -> the first project marker in it, for a list of files. Made once per list: looking through
1172
+ every file for each new module was quadratic (minutes on a repository of tens of thousands of files)."""
1173
+ got = getattr(self, "_marker_memo", None)
1174
+ if got is None or got[0] is not files:
1175
+ out: dict[str, str] = {}
1176
+ for f in files:
1177
+ d, _, base = f.rpartition("/")
1178
+ if d not in out and (f.endswith(".csproj") or base in MODULE_MARKERS):
1179
+ out[d] = base
1180
+ got = self._marker_memo = (files, out)
1181
+ return got[1]
1182
+
1183
+ def _projects(self, files: list[str], repo: Optional[str] = None) -> None:
1184
+ """Project files give exact module-to-module and module-to-package edges."""
1185
+ repo = repo or self.repo
1186
+ root = self.repos[repo]
1187
+ for f in files:
1188
+ if not f.endswith(".csproj"):
1189
+ continue
1190
+ d = f.rpartition("/")[0]
1191
+ mid = f"{repo}:module:{d or '.'}"
1192
+ if mid not in self.nodes:
1193
+ continue
1194
+ try:
1195
+ text = (root / f).read_text(encoding="utf-8", errors="replace")
1196
+ except OSError:
1197
+ continue
1198
+ for ref in re.findall(r'<ProjectReference\s+Include="([^"]+)"', text):
1199
+ target = (Path(d) / ref.replace("\\", "/")).parent
1200
+ tdir = Path(*_normalize(target.parts)).as_posix() if target.parts else ""
1201
+ tid = f"{repo}:module:{tdir or '.'}"
1202
+ if target.parts[:1] == ("..",) or tid not in self.nodes:
1203
+ tid = self._module_elsewhere(root / target) or tid # a project in another repo of the workspace
1204
+ if tid in self.nodes:
1205
+ self.edges.append(Edge("imports", mid, tid, "exact", {"via": "ProjectReference"}))
1206
+ self.project_refs[mid].add(tid)
1207
+ for name, ver in re.findall(r'<PackageReference\s+Include="([^"]+)"(?:\s+Version="([^"]+)")?', text):
1208
+ xid = self._external("nuget", name, {"category": "package", "version": ver or None}, repo)
1209
+ self.edges.append(Edge("depends_on", mid, xid, "exact", {"version_range": ver or None}))
1210
+ sdk = re.search(r'<Project\s+Sdk="([^"/]+)(?:/([^"]+))?"', text)
1211
+ if sdk and sdk.group(1) != "Microsoft.NET.Sdk":
1212
+ xid = self._external("nuget", sdk.group(1), {"category": "sdk", "version": sdk.group(2)}, repo)
1213
+ self.edges.append(Edge("depends_on", mid, xid, "exact", {"version_range": sdk.group(2)}))
1214
+ tf = re.search(r"<TargetFramework>([^<]+)<", text)
1215
+ if tf:
1216
+ self.nodes[mid].attrs["target_framework"] = tf.group(1)
1217
+
1218
+ def _module_elsewhere(self, path: Path) -> Optional[str]:
1219
+ """The module of another workspace repo that an absolute directory is, if any."""
1220
+ if len(self.repos) < 2:
1221
+ return None
1222
+ path = Path(os.path.normpath(path))
1223
+ for repo, root in self.repos.items():
1224
+ if path == root or root in path.parents:
1225
+ mid = f"{repo}:module:{path.relative_to(root).as_posix()}"
1226
+ return mid if mid in self.nodes else None
1227
+ return None
1228
+
1229
+ def _external(self, eco: str, name: str, attrs: Optional[dict] = None, repo: Optional[str] = None) -> str:
1230
+ repo = repo or self.repo
1231
+ xid = f"{repo}:ext:{eco}:{name}"
1232
+ if xid not in self.nodes:
1233
+ self._add(Node(id=xid, kind="external", name=name, parent_id=repo,
1234
+ attrs={"ecosystem": eco, **(attrs or {})}))
1235
+ return xid
1236
+
1237
+ # -- indexes -------------------------------------------------------------
1238
+ def _build_indexes(self) -> None:
1239
+ self.types_by_name: dict[tuple, list[str]] = defaultdict(list) # (lang, name) -> type ids
1240
+ self.members: dict[str, dict[str, list[Node]]] = defaultdict(lambda: defaultdict(list))
1241
+ self.by_name: dict[tuple, list[Node]] = defaultdict(list) # (lang, name) -> callables
1242
+ self.field_type: dict[str, dict[str, str]] = defaultdict(dict) # type id -> field -> type name
1243
+ self.field_types_global: dict[tuple, set] = defaultdict(set) # (lang, field) -> type names
1244
+ self.bases: dict[str, list[str]] = defaultdict(list)
1245
+ self.field_names: dict[str, set] = defaultdict(set)
1246
+ self.ns_modules: dict[str, set] = defaultdict(set) # C# namespace -> module ids
1247
+ self.py_modules: dict[str, dict] = defaultdict(dict) # repo -> python module path -> file id
1248
+ self.path_modules: dict[str, dict] = defaultdict(dict) # repo -> path without extension -> file id (TypeScript)
1249
+ self.py_importable: dict[str, str] = {} # python import name -> file id, across the workspace
1250
+ self.star_exports: dict[str, list] = defaultdict(list) # file -> files it re-exports everything from
1251
+ self.default_export: dict[str, str] = {} # file -> the node it exports as default
1252
+ self.packages: dict[str, dict] = {} # npm package name -> {"dir", "exports", "main"}
1253
+ for n in self.nodes.values():
1254
+ if n.attrs.get("is_default_export") and n.kind in ("callable", "type"):
1255
+ self.default_export[n.parent_id] = n.id
1256
+ self.file_of: dict[str, str] = {}
1257
+ self.field_calls: dict[tuple, str] = {} # (type id, field) -> the method whose result it holds
1258
+ self.py_vars: dict[str, dict] = {fid: res.var_types for fid, res in self.results.items() if res.var_types}
1259
+ for n in self.nodes.values():
1260
+ if n.kind == "type":
1261
+ self.types_by_name[(n.language, n.name)].append(n.id)
1262
+ elif n.kind == "callable":
1263
+ self.members[n.parent_id][n.name].append(n)
1264
+ self.by_name[(n.language, n.name)].append(n)
1265
+ elif n.kind == "field":
1266
+ self.field_names[n.parent_id].add(n.name)
1267
+ if n.attrs.get("type_call"):
1268
+ self.field_calls[(n.parent_id, n.name)] = n.attrs["type_call"]
1269
+ tn = n.attrs.get("type_name")
1270
+ if tn:
1271
+ self.field_type[n.parent_id][n.name] = tn
1272
+ self.field_types_global[(n.language, n.name)].add(tn)
1273
+ for fid, res in self.results.items():
1274
+ lang = self.file_lang[fid]
1275
+ mod_id = self.nodes[fid].parent_id
1276
+ for d in res.declares:
1277
+ if lang == "csharp":
1278
+ self.ns_modules[d].add(mod_id)
1279
+ elif lang == "python":
1280
+ self.py_modules[_repo_of(fid)][d] = fid
1281
+ else:
1282
+ self.path_modules[_repo_of(fid)][d] = fid
1283
+ for n in res.nodes:
1284
+ self.file_of[n.id] = fid
1285
+ # A package under a source root (src/flask) is imported by its own name (flask), not by its path.
1286
+ for repo in self.repos:
1287
+ py_files = {self.nodes[f].path: f for f in self.results if self.file_lang[f] == "python" and _repo_of(f) == repo}
1288
+ dirs_with_init = {p.rsplit("/", 1)[0] if "/" in p else "" for p in py_files if p.endswith("__init__.py")}
1289
+ for path, fid in sorted(py_files.items()):
1290
+ parts = path.split("/")
1291
+ start = len(parts) - 1
1292
+ while start > 0 and "/".join(parts[:start]) in dirs_with_init:
1293
+ start -= 1
1294
+ if start == len(parts) - 1 and "src" in parts[:-1]:
1295
+ # a namespace package (PEP 420, no __init__.py) in a src layout: src/corp/tools/x.py is corp.tools.x
1296
+ start = len(parts) - 1 - parts[:-1][::-1].index("src")
1297
+ if 0 < start < len(parts) - 1:
1298
+ self.py_modules[repo].setdefault(module_path("/".join(parts[start:])), fid)
1299
+ if len(self.repos) > 1 and start < len(parts) - 1:
1300
+ # Inside a package: another repository of the workspace can import it by this name.
1301
+ self.py_importable.setdefault(module_path("/".join(parts[start:])), fid)
1302
+
1303
+ # -- imports -------------------------------------------------------------
1304
+ def _resolve_imports(self) -> None:
1305
+ self.cs_usings: dict[str, set] = defaultdict(set) # file -> namespaces
1306
+ self.cs_static: dict[str, list] = defaultdict(list) # file -> type names from `using static`
1307
+ self.cs_alias: dict[str, dict] = defaultdict(dict) # file -> alias -> type name
1308
+ self.py_names: dict[str, dict] = defaultdict(dict) # file -> local name -> (file id, symbol|None)
1309
+ self.outside_imports: dict[str, set] = defaultdict(set) # file -> local names imported from outside the workspace
1310
+ self.import_targets: dict[str, set] = defaultdict(set) # file -> files it imports, by any form of import
1311
+ for repo, files in self.files_of.items():
1312
+ for f in files:
1313
+ if f.rsplit("/", 1)[-1] == "package.json":
1314
+ try:
1315
+ data = json.loads((self.repos[repo] / f).read_text(encoding="utf-8"))
1316
+ except (OSError, ValueError, RecursionError): # RecursionError: nested past the limit
1317
+ continue
1318
+ if isinstance(data, dict) and data.get("name"):
1319
+ self.packages[data["name"]] = {"dir": f.rpartition("/")[0], "exports": data.get("exports"), "repo": repo,
1320
+ "main": data.get("source") or data.get("module") or data.get("main") or data.get("types")}
1321
+ self._go_modules()
1322
+ seen = set()
1323
+ stdlib = getattr(sys, "stdlib_module_names", frozenset())
1324
+ self.py_local: dict[tuple, tuple] = {} # (function, name) -> what an import inside that function binds
1325
+
1326
+ def bind(fid: str, src: str, name: str, value: tuple) -> None:
1327
+ if src == fid:
1328
+ self.py_names[fid][name] = value
1329
+ return
1330
+ # An import inside a function binds the name there only: elsewhere in the file the same name may be a
1331
+ # local variable (`query = Query()` beside another function's `from .sql import query`).
1332
+ self.py_local[(src, name)] = value
1333
+ if value[0]:
1334
+ self.import_targets[fid].add(value[0])
1335
+ for fid, res in self.results.items():
1336
+ lang = self.file_lang[fid]
1337
+ for imp in res.imports:
1338
+ if lang == "csharp":
1339
+ if imp.alias:
1340
+ self.cs_alias[fid][imp.alias] = imp.target.rsplit(".", 1)[-1]
1341
+ if imp.is_static:
1342
+ self.cs_static[fid].append(imp.target.rsplit(".", 1)[-1])
1343
+ ns = imp.target
1344
+ mods = self.ns_modules.get(ns)
1345
+ if not mods and (imp.is_static or imp.alias):
1346
+ ns = imp.target.rsplit(".", 1)[0]
1347
+ mods = self.ns_modules.get(ns)
1348
+ self.cs_usings[fid].add(ns)
1349
+ vis = self._visible(fid)
1350
+ declared = bool(mods)
1351
+ if mods and vis is not None:
1352
+ # A namespace can be declared by several projects; only the referenced ones count.
1353
+ mods = {m for m in mods if m in vis or m == self.nodes[fid].parent_id}
1354
+ if mods:
1355
+ for m in sorted(mods):
1356
+ key = (fid, m, ns)
1357
+ if key not in seen and m != self.nodes[fid].parent_id:
1358
+ seen.add(key)
1359
+ self.edges.append(Edge("imports", fid, m, "exact", {"namespace": ns}))
1360
+ elif not declared:
1361
+ xid = self._external("dotnet", imp.target if not (imp.is_static or imp.alias) else ns,
1362
+ {"category": "namespace"}, _repo_of(fid))
1363
+ if (fid, xid) not in seen:
1364
+ seen.add((fid, xid))
1365
+ self.edges.append(Edge("imports", fid, xid, "exact"))
1366
+ else:
1367
+ target = self._find_module(fid, imp.target)
1368
+ on_path = False
1369
+ if not target and lang == "python" and imp.target.split(".")[0] not in stdlib:
1370
+ target = self._py_on_path(fid, imp.target)
1371
+ on_path = bool(target)
1372
+ subs = {} if target or lang != "python" else \
1373
+ {s: self._py_module(fid, f"{imp.target}.{s.partition(' as ')[0]}") for s in imp.symbols or ()}
1374
+ if any(subs.values()):
1375
+ # `from corp.tools import strings` where corp/tools is a namespace package (no __init__.py):
1376
+ # the names are its modules
1377
+ for s, sub in subs.items():
1378
+ if sub:
1379
+ name, _, alias = s.partition(" as ")
1380
+ bind(fid, imp.src_id, alias or name, (sub, None))
1381
+ self.import_targets[fid].add(sub)
1382
+ if (fid, sub) not in seen and sub != fid:
1383
+ seen.add((fid, sub))
1384
+ self.edges.append(Edge("imports", fid, sub, "exact", {"symbols": imp.symbols}))
1385
+ elif target:
1386
+ if imp.symbols:
1387
+ for s in imp.symbols:
1388
+ name, _, alias = s.partition(" as ")
1389
+ if name == "*":
1390
+ self.star_exports[fid].append(target)
1391
+ continue
1392
+ sub = self._py_module(fid, f"{imp.target}.{name}") if lang == "python" else None
1393
+ bind(fid, imp.src_id, alias or name, (sub, None) if sub else (target, name))
1394
+ elif imp.alias or lang == "python":
1395
+ bind(fid, imp.src_id, imp.alias or imp.target, (target, None))
1396
+ self.import_targets[fid].add(target)
1397
+ if (fid, target) not in seen and target != fid:
1398
+ seen.add((fid, target))
1399
+ self.edges.append(Edge("imports", fid, target, "heuristic" if on_path else "exact",
1400
+ {"symbols": imp.symbols, **({"found_by": "sys.path"} if on_path else {})}))
1401
+ elif lang == "python":
1402
+ top = imp.target.split(".")[0] or imp.target
1403
+ xid = self._external("python", top,
1404
+ {"category": "stdlib" if top in stdlib else "package"}, _repo_of(fid))
1405
+ if top: # `import sqlite3`: sqlite3.connect() is not ours to resolve
1406
+ self.outside_imports[fid].update(
1407
+ [(s.partition(" as ")[2] or s) for s in imp.symbols] if imp.symbols else [imp.alias or top])
1408
+ if (fid, xid) not in seen:
1409
+ seen.add((fid, xid))
1410
+ self.edges.append(Edge("imports", fid, xid, "exact", {"symbols": imp.symbols}))
1411
+ elif lang != "typescript" and not imp.target.startswith("."):
1412
+ # Go, Java, Rust...: the module or package path as written (go:example.com/app/store), not npm's.
1413
+ xid = self._external(lang, imp.target.strip("\"'<>` "), {"category": "package"}, _repo_of(fid))
1414
+ if (fid, xid) not in seen:
1415
+ seen.add((fid, xid))
1416
+ self.edges.append(Edge("imports", fid, xid, "exact", {"symbols": imp.symbols}))
1417
+ elif not imp.target.startswith("."):
1418
+ spec = imp.target[5:] if imp.target.startswith("node:") else imp.target
1419
+ parts = spec.split("/")
1420
+ top = "/".join(parts[:2]) if spec.startswith("@") else parts[0]
1421
+ builtin = imp.target.startswith("node:") or top in NODE_BUILTINS
1422
+ xid = self._external("npm", top, {"category": "stdlib" if builtin else "package"}, _repo_of(fid))
1423
+ for s in imp.symbols: # a name from outside: calls to it are not ours to resolve
1424
+ self.outside_imports[fid].add(s.partition(" as ")[2] or s)
1425
+ if imp.alias:
1426
+ self.outside_imports[fid].add(imp.alias)
1427
+ if (fid, xid) not in seen:
1428
+ seen.add((fid, xid))
1429
+ self.edges.append(Edge("imports", fid, xid, "exact", {"symbols": imp.symbols}))
1430
+
1431
+ def _script_entries(self) -> None:
1432
+ """A TypeScript file that does work at its top level and that nothing imports is a program:
1433
+ a server's start file, a build script. Tests and config files are not."""
1434
+ imported = {t for targets in self.import_targets.values() for t in targets}
1435
+ for fid, res in self.results.items():
1436
+ if self.file_lang[fid] != "typescript" or fid in imported:
1437
+ continue
1438
+ path = self.nodes[fid].path
1439
+ base = path.rsplit("/", 1)[-1]
1440
+ top = f"{_repo_of(fid)}:typescript:{self._modpath(fid)}.<module>"
1441
+ if top not in self.nodes or top + "#entry" in self.nodes or re.search(r"\.(test|spec|bench|config|setup|d)\.", base) \
1442
+ or "/test/" in "/" + path or "/__tests__/" in "/" + path or any(n.kind == "test" for n in res.nodes):
1443
+ continue
1444
+ calls = [c for c in self._open(fid, res).calls if c.src_id == top and not c.ref]
1445
+ if len(calls) < 1:
1446
+ continue
1447
+ ep = Node(id=top + "#entry", kind="entry_point", name=path, parent_id=fid, language="typescript", path=path,
1448
+ span_start=1, span_end=self.nodes[fid].span_end, attrs={"trigger": "cli", "address": path, "found_by": "nothing imports it"})
1449
+ self._add(ep)
1450
+ self.file_of[ep.id] = fid
1451
+ self.edges.append(Edge("exposes", ep.id, top))
1452
+
1453
+ def _attach_methods(self) -> None:
1454
+ """A method written outside its type's body (Go's func (t *T) Run(), Rust's impl T { ... } in another
1455
+ file) belongs to the type of that name, looked for in the same directory first."""
1456
+ types: dict[tuple, list[str]] = defaultdict(list)
1457
+ for n in self.nodes.values():
1458
+ if n.kind == "type":
1459
+ types[(n.language, n.name)].append(n.id)
1460
+ for n in list(self.nodes.values()):
1461
+ owner = n.attrs.get("owner_name") if n.kind == "callable" else None
1462
+ if not owner or (n.parent_id in self.nodes and self.nodes[n.parent_id].kind == "type"):
1463
+ continue
1464
+ cands = types.get((n.language, owner), [])
1465
+ here = (_repo_of(n.id), (n.path or "").rpartition("/")[0])
1466
+ near = [t for t in cands if (_repo_of(t), (self.nodes[t].path or "").rpartition("/")[0]) == here] or cands
1467
+ if len(near) == 1:
1468
+ n.parent_id = near[0]
1469
+ n.attrs["type_id"] = near[0]
1470
+ fid = f"{_repo_of(n.id)}:file:{n.path}"
1471
+ for res in (self.results.get(fid),):
1472
+ for c in (self._open(fid, res).calls if res else ()):
1473
+ if c.src_id == n.id and c.enclosing_type is None:
1474
+ c.enclosing_type = near[0]
1475
+ if self._opened is not None:
1476
+ self._opened[2] = True # packed again when it is let go
1477
+
1478
+ def _go_modules(self) -> None:
1479
+ """Go modules of the workspace: each go.mod's `module` path, at the go.mod's directory (a repository may
1480
+ hold several, a go.work tying them), and its `replace` directives that point at a local directory."""
1481
+ self.go_mods: dict[str, tuple] = {} # module path -> (repo, directory in that repo)
1482
+ replaces = []
1483
+ for repo, files in self.files_of.items():
1484
+ for f in files:
1485
+ if f.rsplit("/", 1)[-1] != "go.mod":
1486
+ continue
1487
+ try:
1488
+ text = (self.repos[repo] / f).read_text(encoding="utf-8", errors="replace")
1489
+ except OSError:
1490
+ continue
1491
+ here = f.rpartition("/")[0]
1492
+ text = re.sub(r"//[^\n]*", "", text)
1493
+ m = re.search(r"^\s*module\s+\"?([^\s\"]+)", text, re.M)
1494
+ if m:
1495
+ self.go_mods.setdefault(m.group(1), (repo, here))
1496
+ # replace example.com/shared => ../shared, alone or in a replace ( ... ) block
1497
+ for r in re.finditer(r"^\s*(?:replace\s+)?([^\s(]+)(?:\s+v[^\s]+)?\s+=>\s+(\.{1,2}/[^\s]*|\.{1,2})\s*$",
1498
+ text, re.M):
1499
+ local = posixpath.normpath(posixpath.join(here, r.group(2)))
1500
+ if not local.startswith(".."):
1501
+ replaces.append((r.group(1), (repo, "" if local == "." else local)))
1502
+ for path, at in replaces:
1503
+ self.go_mods.setdefault(path, at)
1504
+
1505
+ def _go_import(self, fid: str, target: str) -> Optional[list]:
1506
+ """The files of the package a Go import names, when it is under a module of the workspace: example.com/app/store
1507
+ with `module example.com/app` is the directory store/ beside that go.mod. None for a path under no module."""
1508
+ t = target.strip().strip('"`')
1509
+ mod = max((m for m in getattr(self, "go_mods", {}) if t == m or t.startswith(m + "/")), key=len, default=None)
1510
+ if mod is None:
1511
+ return None
1512
+ repo, base = self.go_mods[mod]
1513
+ d = posixpath.join(base, t[len(mod):].lstrip("/")).strip("/") if t != mod else base
1514
+ return sorted(f for f in self.results if _repo_of(f) == repo and self.file_lang[f] == "go"
1515
+ and self.nodes[f].path.rpartition("/")[0] == d)
1516
+
1517
+ def _find_module(self, fid: str, target: str) -> Optional[str]:
1518
+ if self.file_lang[fid] == "go":
1519
+ hits = self._go_import(fid, target)
1520
+ if hits is not None:
1521
+ for h in hits[1:]:
1522
+ self.import_targets[fid].add(h)
1523
+ return hits[0] if hits else None
1524
+ if getattr(BY_LANGUAGE.get(self.file_lang[fid]), "GENERIC", False):
1525
+ hits = self._generic_import(fid, target)
1526
+ for h in hits[1:]:
1527
+ self.import_targets[fid].add(h)
1528
+ return hits[0] if hits else None
1529
+ if self.file_lang[fid] == "python":
1530
+ return self._py_module(fid, target)
1531
+ hit = self._path_module(target, _repo_of(fid))
1532
+ if hit is None and self.file_lang[fid] == "typescript" and not target.startswith("."):
1533
+ hit = self._ts_alias(fid, target)
1534
+ return hit
1535
+
1536
+ def _ts_alias(self, fid: str, target: str) -> Optional[str]:
1537
+ """An import through compilerOptions.paths of the nearest tsconfig.json or jsconfig.json above the file
1538
+ (`@ui/*` -> `src/*`), resolved against its baseUrl, or its own directory when it has none. `extends` is not
1539
+ followed: a tsconfig with no paths of its own lets the import fall to the next one up, as one that extends
1540
+ a shared base mostly does."""
1541
+ repo = _repo_of(fid)
1542
+ configs = self.__dict__.setdefault("_ts_configs", {})
1543
+ if repo not in configs:
1544
+ configs[repo] = found = {}
1545
+ for f in self.files_of.get(repo, ()):
1546
+ name = f.rsplit("/", 1)[-1]
1547
+ if not (name in ("tsconfig.json", "jsconfig.json") or re.fullmatch(r"tsconfig\.[\w.-]+\.json", name)):
1548
+ continue
1549
+ opts = (_jsonc(self.repos[repo] / f) or {}).get("compilerOptions")
1550
+ paths = opts.get("paths") if isinstance(opts, dict) else None
1551
+ if isinstance(paths, dict):
1552
+ here = f.rpartition("/")[0]
1553
+ base = posixpath.normpath(posixpath.join(here, str(opts.get("baseUrl") or ".")))
1554
+ found.setdefault(here, []).append(("" if base == "." else base, paths))
1555
+ here = self.nodes[fid].path.rpartition("/")[0]
1556
+ while True:
1557
+ for base, paths in configs[repo].get(here, ()):
1558
+ fits = []
1559
+ for key, subs in paths.items():
1560
+ pre, star, post = key.partition("*")
1561
+ if (target == key) if not star else (target.startswith(pre) and target.endswith(post)
1562
+ and len(target) >= len(pre) + len(post)):
1563
+ fits.append((len(pre), target[len(pre):len(target) - len(post)] if star else "", subs))
1564
+ for _, middle, subs in sorted(fits, key=lambda x: -x[0]): # the longest prefix wins, as in tsc
1565
+ for sub in subs if isinstance(subs, list) else [subs]:
1566
+ if isinstance(sub, str):
1567
+ path = posixpath.normpath(posixpath.join(base, sub.replace("*", middle, 1)))
1568
+ hit = self._path_module("./" + path, repo)
1569
+ if hit:
1570
+ return hit
1571
+ if fits:
1572
+ return None
1573
+ if not here:
1574
+ return None
1575
+ here = here.rpartition("/")[0]
1576
+
1577
+ def _generic_import(self, fid: str, target: str) -> list[str]:
1578
+ """Files an import names, whatever the language writes: a relative path (./x, ../x.h), a dotted or ::
1579
+ path (com.foo.Bar, crate::a::b), or a package path ("github.com/x/y/pkg" names a directory)."""
1580
+ if not hasattr(self, "_suffixes"):
1581
+ self._suffixes: dict[str, list[str]] = defaultdict(list)
1582
+ self._dirs: dict[str, list[str]] = defaultdict(list)
1583
+ for f in self.results:
1584
+ path = self.nodes[f].path
1585
+ stem = re.sub(r"\.[A-Za-z0-9]+$", "", path)
1586
+ parts = stem.split("/")
1587
+ for i in range(len(parts)):
1588
+ self._suffixes["/".join(parts[i:])].append(f)
1589
+ if parts[-1] in ("mod", "index", "__init__", "lib") and i < len(parts) - 1:
1590
+ self._suffixes["/".join(parts[i:-1])].append(f)
1591
+ d = parts[:-1]
1592
+ for i in range(len(d)):
1593
+ self._dirs["/".join(d[i:])].append(f)
1594
+ lang, repo = self.file_lang[fid], _repo_of(fid)
1595
+ here = self.nodes[fid].path.rpartition("/")[0]
1596
+ t = target.strip().strip("\"'<>`")
1597
+ if not t:
1598
+ return []
1599
+ m = re.fullmatch(r"(\.+)([\w.]*)", t)
1600
+ if m: # Python's relative import: one dot is this package, each further dot one level up
1601
+ base = here
1602
+ for _ in range(len(m.group(1)) - 1):
1603
+ base = base.rpartition("/")[0]
1604
+ t = "./" + posixpath.join(posixpath.relpath(base or ".", here or "."), m.group(2).replace(".", "/")) if m.group(2) else "./"
1605
+ if not m.group(2):
1606
+ return [f for f in self._suffixes.get(posixpath.join(base, "__init__") if base else "__init__", []) if _repo_of(f) == repo][:1]
1607
+ if t.startswith("."):
1608
+ joined = posixpath.normpath(posixpath.join(here, t))
1609
+ key = re.sub(r"\.[A-Za-z0-9]+$", "", joined)
1610
+ return [f for f in self._suffixes.get(key, []) if self.nodes[f].path.startswith(key) and _repo_of(f) == repo] or self._dirs.get(key, [])[:0]
1611
+ t = re.sub(r"^(crate|self|super)::", "", t)
1612
+ key = re.sub(r"(::|\.|\\)", "/", t) if not "/" in t else re.sub(r"\.[A-Za-z0-9]+$", "", t)
1613
+ key = key.rstrip("/*").strip("/")
1614
+ def find(accept):
1615
+ same_lang = lambda fs, k: [f for f in fs if self.file_lang[f] == lang and accept(f, k)]
1616
+ for k in (key, key.rsplit("/", 1)[0] if "/" in key else None): # import a.b.C names a file a/b/C, or a symbol C in a/b
1617
+ if not k:
1618
+ continue
1619
+ hits = same_lang(self._suffixes.get(k, []), k)
1620
+ if hits:
1621
+ return hits[:8] if len(hits) <= 8 else []
1622
+ hits = same_lang(self._dirs.get(k, []), k) # a package or module directory
1623
+ if hits and len({self.nodes[h].path.rpartition("/")[0] for h in hits}) == 1:
1624
+ return hits[:40]
1625
+ return []
1626
+ own = find(lambda f, k: _repo_of(f) == repo)
1627
+ if own or len(self.repos) == 1:
1628
+ return own
1629
+ # Another repository of the workspace: only a name that starts at its root or at a source root there
1630
+ # (werkzeug/routing in src/werkzeug/routing.py), not any file that happens to end the same way (json).
1631
+ return find(lambda f, k: _repo_of(f) != repo and self._rooted(self.nodes[f].path, k))
1632
+
1633
+ SOURCE_ROOTS = {"src", "lib", "source", "sources", "main", "java", "kotlin", "scala", "pkg", "include", "app"}
1634
+
1635
+ def _rooted(self, path: str, key: str) -> bool:
1636
+ """Whether `key` names `path` from the top of its repository or of a source root in it."""
1637
+ stem = re.sub(r"\.[A-Za-z0-9]+$", "", path)
1638
+ for tail in ("", "/mod", "/index", "/__init__", "/lib"):
1639
+ for cand in (stem, stem.rpartition("/")[0]): # the file itself, or its directory (a package)
1640
+ if tail and not cand.endswith(tail):
1641
+ continue
1642
+ cand = cand[: len(cand) - len(tail)] if tail else cand
1643
+ if cand == key or cand.endswith("/" + key):
1644
+ prefix = cand[: len(cand) - len(key)].strip("/")
1645
+ if not prefix or all(p in self.SOURCE_ROOTS for p in prefix.split("/")):
1646
+ return True
1647
+ return False
1648
+
1649
+ def _path_module(self, target: str, repo: Optional[str] = None) -> Optional[str]:
1650
+ """A TypeScript import: ./path from the repo root (the adapter resolved it), or a workspace package,
1651
+ which may live in another repository of the workspace."""
1652
+ def at(path, repo=repo or self.repo):
1653
+ # only a leading ./ goes: lstrip("./") also took the dot off a directory such as .storybook
1654
+ path = re.sub(r"\.(d\.[cm]?ts|[cm]?[jt]sx?)$", "", path[2:] if path.startswith("./") else path)
1655
+ return self.path_modules[repo].get(path) or self.path_modules[repo].get(path + "/index")
1656
+ if target.startswith("."):
1657
+ return at(target[2:] if target.startswith("./") else target)
1658
+ name = max((p for p in self.packages if target == p or target.startswith(p + "/")), key=len, default=None)
1659
+ if name is None:
1660
+ return None
1661
+ pkg, sub = self.packages[name], target[len(name):].lstrip("/")
1662
+ exports = pkg["exports"]
1663
+ entry = exports.get("./" + sub if sub else ".") if isinstance(exports, dict) else (exports if not sub else None)
1664
+ while isinstance(entry, dict): # {"import": ..., "types": ..., "default": ...}
1665
+ entry = next((entry[k] for k in ("source", "import", "default", "types", "require") if k in entry), None)
1666
+ tries = [entry] if isinstance(entry, str) else []
1667
+ tries += [f"{sub}", f"src/{sub}"] if sub else [pkg["main"], "src/index", "index"]
1668
+ for t in tries:
1669
+ if t:
1670
+ hit = at(posixpath.normpath(posixpath.join(pkg["dir"], t)), pkg["repo"])
1671
+ if hit:
1672
+ return hit
1673
+ return None
1674
+
1675
+ def _py_module(self, fid: str, target: str) -> Optional[str]:
1676
+ repo = _repo_of(fid)
1677
+ mods = self.py_modules[repo]
1678
+ if target in mods:
1679
+ return mods[target]
1680
+ own = module_path(self.nodes[fid].path)
1681
+ pkg = own.rsplit(".", 1)[0] if "." in own else ""
1682
+ sibling = f"{pkg}.{target}" if pkg else target
1683
+ if sibling in mods: # script-style import of a file in the same directory
1684
+ return mods[sibling]
1685
+ hit = self.py_importable.get(target) # a package of another repository in the workspace
1686
+ return hit if hit and _repo_of(hit) != repo else None
1687
+
1688
+ def _py_on_path(self, fid: str, target: str) -> Optional[str]:
1689
+ """`sys.path.insert(0, <dir>)` then `import validate`: a script or a test that puts a directory on the path
1690
+ imports a file there by its bare name. When the importing file, or a conftest.py above it, changes sys.path,
1691
+ an import nothing else resolves is taken to be the repository's one module of that name (or the one in a
1692
+ directory above the importer, when there are several). A heuristic: which directory is added is not worked
1693
+ out, so the import edge says so."""
1694
+ if target.startswith(".") or not self._edits_sys_path(fid):
1695
+ return None
1696
+ repo = _repo_of(fid)
1697
+ tail = "." + target
1698
+ cands = sorted({f for m, f in self.py_modules[repo].items() if (m == target or m.endswith(tail)) and f != fid})
1699
+ if len(cands) > 1:
1700
+ here = self.nodes[fid].path.rpartition("/")[0]
1701
+ cands = [f for f in cands if (here + "/").startswith(self.nodes[f].path.rpartition("/")[0].rstrip("/") + "/")
1702
+ or "/" not in self.nodes[f].path]
1703
+ return cands[0] if len(cands) == 1 else None
1704
+
1705
+ def _edits_sys_path(self, fid: str) -> bool:
1706
+ cache = self.__dict__.setdefault("_sys_path_files", {})
1707
+ if fid not in cache:
1708
+ repo, path = _repo_of(fid), self.nodes[fid].path
1709
+ parts = path.split("/")
1710
+ files = [path] + ["/".join(parts[:i] + ["conftest.py"]) for i in range(len(parts) - 1, -1, -1)]
1711
+ hit = False
1712
+ for p in dict.fromkeys(files):
1713
+ if p != path and module_path(p) not in self.py_modules[repo]:
1714
+ continue
1715
+ try:
1716
+ hit = b"sys.path" in read_source(self.repos[repo] / p)
1717
+ except OSError:
1718
+ hit = False
1719
+ if hit:
1720
+ break
1721
+ cache[fid] = hit
1722
+ return cache[fid]
1723
+
1724
+ def _modpath(self, fid: str) -> str:
1725
+ adapter = BY_LANGUAGE[self.file_lang[fid]]
1726
+ if fid in self.keep_ext:
1727
+ return adapter.module_path(self.nodes[fid].path, True)
1728
+ return adapter.module_path(self.nodes[fid].path)
1729
+
1730
+ def _py_export(self, target_fid: str, symbol: str, depth: int = 0) -> Optional[str]:
1731
+ """The node a module exposes under a name, following re-exports (`from .app import Flask`,
1732
+ `export { a } from "./b"`, `export * from "./c"`)."""
1733
+ if target_fid not in self.file_lang:
1734
+ return None
1735
+ if symbol == "default" and target_fid in self.default_export:
1736
+ return self.default_export[target_fid]
1737
+ cand = f"{_repo_of(target_fid)}:{self.file_lang[target_fid]}:{self._modpath(target_fid)}.{symbol}"
1738
+ if cand in self.nodes:
1739
+ return cand
1740
+ if depth < 6 and symbol in self.py_names.get(target_fid, {}):
1741
+ nxt, sub = self.py_names[target_fid][symbol]
1742
+ if sub is not None:
1743
+ return self._py_export(nxt, sub, depth + 1)
1744
+ if depth < 6:
1745
+ for nxt in self.star_exports.get(target_fid, ()):
1746
+ hit = self._py_export(nxt, symbol, depth + 1)
1747
+ if hit:
1748
+ return hit
1749
+ return None
1750
+
1751
+ def _export_module(self, target_fid: str, symbol: str, depth: int = 0) -> Optional[str]:
1752
+ """The file a module exposes whole under a name (`export * as ns from './format'`, `import util as u`),
1753
+ following re-exports, or None."""
1754
+ if depth > 6:
1755
+ return None
1756
+ bound = self.py_names.get(target_fid, {}).get(symbol)
1757
+ if bound:
1758
+ nxt, sub = bound
1759
+ return nxt if sub is None else self._export_module(nxt, sub, depth + 1)
1760
+ for nxt in self.star_exports.get(target_fid, ()):
1761
+ hit = self._export_module(nxt, symbol, depth + 1)
1762
+ if hit:
1763
+ return hit
1764
+ return None
1765
+
1766
+ def _py_name(self, fid: str, src: str, name: str) -> tuple:
1767
+ """What a name imported in a file means inside one of its functions: an import in the function or an
1768
+ enclosing one first, then the file's. (None, None) when it cannot be told."""
1769
+ cur = src
1770
+ while cur in self.nodes and self.nodes[cur].kind == "callable":
1771
+ if (cur, name) in self.py_local:
1772
+ return self.py_local[(cur, name)]
1773
+ cur = self.nodes[cur].parent_id
1774
+ return self.py_names[fid].get(name, (None, None))
1775
+
1776
+ def _py_bound(self, fid: str, src: Optional[str], name: str) -> bool:
1777
+ """Does an import bind this name where src runs: in src, a function around it, or the file?"""
1778
+ cur = src
1779
+ while cur in self.nodes and self.nodes[cur].kind == "callable":
1780
+ if (cur, name) in self.py_local:
1781
+ return True
1782
+ cur = self.nodes[cur].parent_id
1783
+ return name in self.py_names[fid]
1784
+
1785
+ def _py_var(self, fid: str, name: str, depth: int = 0) -> Optional[str]:
1786
+ """The type of a module-level variable (`g: Globals = ...`), found through imports and re-exports."""
1787
+ tn = self.py_vars.get(fid, {}).get(name)
1788
+ if tn:
1789
+ return self._type("python", tn, None, file=fid)
1790
+ if depth < 6 and name in self.py_names.get(fid, {}):
1791
+ nxt, sub = self.py_names[fid][name]
1792
+ if sub is not None:
1793
+ return self._py_var(nxt, sub, depth + 1)
1794
+ if depth < 6:
1795
+ for nxt in self.star_exports.get(fid, ()):
1796
+ hit = self._py_var(nxt, name, depth + 1)
1797
+ if hit:
1798
+ return hit
1799
+ return None
1800
+
1801
+ def _attr_type(self, lang: str, inner: CallSite) -> Optional[tuple]:
1802
+ """The type of an attribute read (`app.config`, `flask.g`) used as a receiver: the declared type of the
1803
+ field, of a property's result, or of a module's variable. None when it cannot be told."""
1804
+ fid = self.file_of.get(inner.src_id)
1805
+ if lang == "python" and fid and inner.receiver_type is None and inner.chain is None and self._py_bound(fid, inner.src_id, inner.receiver or ""):
1806
+ target, symbol = self._py_name(fid, inner.src_id, inner.receiver)
1807
+ if target and symbol is None:
1808
+ tid = self._py_var(target, inner.name)
1809
+ return (tid, True) if tid else None
1810
+ tid, _ = self._receiver_type(lang, inner)
1811
+ for t in self._chain(tid):
1812
+ tn = self.field_type.get(t, {}).get(inner.name)
1813
+ if tn:
1814
+ got = self._type(lang, tn, f"{t}.{inner.name}", file=self.file_of.get(t))
1815
+ return (got, True) if got else None
1816
+ getters = [m for m in self.members.get(t, {}).get(inner.name, ()) if m.attrs.get("returns")
1817
+ and any("property" in d for d in m.attrs.get("decorators") or [])]
1818
+ if getters:
1819
+ got = self._type(lang, getters[0].attrs["returns"], getters[0].id)
1820
+ return (got, True) if got else None
1821
+ return None
1822
+
1823
+ def _py_fixtures(self) -> None:
1824
+ """pytest passes a test each fixture named by its parameters. Link them, and give the
1825
+ parameter the fixture's return type so calls on it can be resolved."""
1826
+ self.py_param_type: dict[tuple, str] = {}
1827
+ by_file: dict[str, dict[str, Node]] = defaultdict(dict)
1828
+ for n in self.nodes.values():
1829
+ if n.language == "python" and n.kind == "callable" and n.attrs.get("is_fixture"):
1830
+ by_file[self.file_of[n.id]][n.name] = n
1831
+ if not by_file:
1832
+ return
1833
+ conftests = {(_repo_of(f), self.nodes[f].path.rsplit("/", 1)[0] if "/" in self.nodes[f].path else ""): f
1834
+ for f in by_file if self.nodes[f].path.rsplit("/", 1)[-1] == "conftest.py"}
1835
+
1836
+ def find(fid: str, name: str, skip: Optional[str] = None) -> Optional[Node]:
1837
+ hit = by_file.get(fid, {}).get(name)
1838
+ if hit is not None and hit.id != skip:
1839
+ return hit
1840
+ d = self.nodes[fid].path.rsplit("/", 1)[0] if "/" in self.nodes[fid].path else ""
1841
+ while True:
1842
+ hit = by_file.get(conftests.get((_repo_of(fid), d), ""), {}).get(name)
1843
+ if hit is not None and hit.id != skip:
1844
+ return hit
1845
+ if not d:
1846
+ return None
1847
+ d = d.rsplit("/", 1)[0] if "/" in d else ""
1848
+ memo: dict[str, Optional[str]] = {}
1849
+
1850
+ def returns(fx: Node, depth: int = 0) -> Optional[str]:
1851
+ if fx.id in memo or depth > 4:
1852
+ return memo.get(fx.id)
1853
+ memo[fx.id] = None
1854
+ out = None
1855
+ if fx.attrs.get("returns"):
1856
+ out = self._type("python", fx.attrs["returns"], fx.id)
1857
+ elif fx.attrs.get("returns_call"):
1858
+ recv, method = fx.attrs["returns_call"]
1859
+ dep = find(self.file_of[fx.id], recv, fx.id) if recv in (fx.attrs.get("params") or []) else None
1860
+ owner = returns(dep, depth + 1) if dep is not None else None
1861
+ for m in self._methods(owner, method, 0) if owner else []:
1862
+ if m.attrs.get("returns"):
1863
+ out = self._type("python", m.attrs["returns"], m.id)
1864
+ memo[fx.id] = out
1865
+ return out
1866
+ for n in list(self.nodes.values()):
1867
+ if n.language != "python" or n.kind != "callable" or not (n.attrs.get("is_test") or n.attrs.get("is_fixture")):
1868
+ continue
1869
+ for pname in n.attrs.get("params") or []:
1870
+ fx = find(self.file_of[n.id], pname, n.id)
1871
+ if fx is None:
1872
+ continue
1873
+ self.calls.append((n.id, fx.id, "fixture", "heuristic", n.span_start))
1874
+ tid = returns(fx)
1875
+ if tid:
1876
+ self.py_param_type[(n.id, pname)] = tid
1877
+
1878
+ # -- visibility ----------------------------------------------------------
1879
+ def _visible(self, fid: str) -> Optional[set]:
1880
+ """Modules whose symbols a file can reference, or None when that is not knowable."""
1881
+ mod = self.nodes[fid].parent_id
1882
+ if self.file_lang[fid] in FILE_MODULE:
1883
+ if fid not in self._vis_cache:
1884
+ out, queue = {fid}, [t for t, _ in self.py_names[fid].values() if t] + list(self.star_exports.get(fid, ()))
1885
+ if self.file_lang[fid] == "python":
1886
+ queue += list(self.import_targets.get(fid, ())) # imports inside functions count too
1887
+ while queue: # a file re-exported with `export *` is seen through the file that re-exports it
1888
+ cur = queue.pop()
1889
+ if cur not in out:
1890
+ out.add(cur)
1891
+ queue.extend(self.star_exports.get(cur, ()))
1892
+ self._vis_cache[fid] = out
1893
+ return self._vis_cache[fid]
1894
+ if not (self.nodes[mod].attrs.get("marker") or "").endswith(".csproj"):
1895
+ return None # a loose .cs file: no project file says what it can see
1896
+ if mod not in self._vis_cache:
1897
+ seen, queue = {mod}, [mod]
1898
+ while queue:
1899
+ cur = queue.pop()
1900
+ for nxt in self.project_refs.get(cur, ()):
1901
+ if nxt not in seen:
1902
+ seen.add(nxt)
1903
+ queue.append(nxt)
1904
+ self._vis_cache[mod] = seen
1905
+ return self._vis_cache[mod]
1906
+
1907
+ def _file_set(self, key: tuple) -> Optional[set]:
1908
+ """A set kept for the file whose calls are being resolved. Calls are resolved a file at a time, so only the
1909
+ last few files' sets are kept: keeping one per file held millions of members on a large repository (every
1910
+ Go file sees each file of every package it imports). Import targets are settled before any of these is
1911
+ asked for, so a set made again is the same set."""
1912
+ return self._sets.get(key)
1913
+
1914
+ def _keep_set(self, key: tuple, value: set) -> set:
1915
+ if len(self._sets) >= 16:
1916
+ self._sets.clear()
1917
+ self._sets[key] = value
1918
+ return value
1919
+
1920
+ def _reach(self, fid: str) -> set:
1921
+ """Files reachable through imports. A name found nowhere else in that set is a fair guess."""
1922
+ key = ("reach", fid)
1923
+ got = self._file_set(key)
1924
+ if got is None:
1925
+ out, queue = set(), [fid]
1926
+ while queue:
1927
+ cur = queue.pop()
1928
+ if cur in out:
1929
+ continue
1930
+ out.add(cur)
1931
+ queue.extend(t for t in self.import_targets.get(cur, ()) if t not in out)
1932
+ got = self._keep_set(key, out)
1933
+ return got
1934
+
1935
+ def _can_see(self, fid: str, node_id: str) -> bool:
1936
+ vis = self._visible(fid)
1937
+ if vis is None:
1938
+ return True
1939
+ target_file = self.file_of.get(node_id)
1940
+ if self.file_lang[fid] in FILE_MODULE:
1941
+ return target_file in vis
1942
+ return target_file is not None and self.nodes[target_file].parent_id in vis
1943
+
1944
+ # -- types ---------------------------------------------------------------
1945
+ def _type(self, lang: str, name: str, from_id: Optional[str], file: Optional[str] = None) -> Optional[str]:
1946
+ fid = file or self.file_of.get(from_id or "")
1947
+ arity = None
1948
+ if lang == "python" and "." in name:
1949
+ # flask.Flask: the class a module imported under that name exposes.
1950
+ head, _, rest = name.rpartition(".")
1951
+ mod = self._py_name(fid, from_id, head) if fid and "." not in head else None
1952
+ cand = self._py_export(mod[0], rest) if mod and mod[0] and mod[1] is None else None
1953
+ if cand and self.nodes[cand].kind == "type":
1954
+ return cand
1955
+ name = rest
1956
+ if lang == "csharp":
1957
+ name, tick, count = name.partition("`")
1958
+ arity = int(count) if tick and count.isdigit() else 0
1959
+ if fid:
1960
+ name = self.cs_alias[fid].get(name, name)
1961
+ if lang == "python" and from_id:
1962
+ cur = from_id # a class defined in the function itself or one around it
1963
+ while cur in self.nodes and self.nodes[cur].kind in ("callable", "test", "type"):
1964
+ hit = self.nodes.get(f"{cur}.{name}")
1965
+ if hit is not None and hit.kind == "type" and hit.id != from_id: # `class App(web.App)` is not its own base
1966
+ return hit.id
1967
+ cur = self.nodes[cur].parent_id
1968
+ if lang in FILE_MODULE and fid and (self._py_bound(fid, from_id, name) if lang == "python" else name in self.py_names[fid]):
1969
+ target, symbol = self._py_name(fid, from_id, name) if lang == "python" else self.py_names[fid][name]
1970
+ cand = self._py_export(target, symbol or name)
1971
+ if cand and self.nodes[cand].kind == "type":
1972
+ return cand
1973
+ if lang == "typescript" and fid:
1974
+ if name in self.outside_imports[fid]:
1975
+ return None
1976
+ own = f"{_repo_of(fid)}:{lang}:{self._modpath(fid)}.{name}"
1977
+ if own in self.nodes and self.nodes[own].kind == "type":
1978
+ return own
1979
+ cands = self.types_by_name.get((lang, name), [])
1980
+ if lang == "python":
1981
+ # A class defined inside a function is seen only there (found above), not by name from elsewhere.
1982
+ cands = [c for c in cands if self.nodes[self.nodes[c].parent_id].kind not in ("callable", "test")]
1983
+ if lang == "csharp" and fid:
1984
+ cands = [c for c in cands if self._can_see(fid, c)]
1985
+ if arity is not None and cands:
1986
+ # Foo and Foo<T> are different types. A plain name falls back to the generic one, because
1987
+ # some callers (a receiver's inferred type) do not carry the type arguments.
1988
+ exact = [c for c in cands if _arity(c) == arity]
1989
+ cands = exact or ([] if arity else cands)
1990
+ if not cands:
1991
+ return None
1992
+ if len(cands) == 1:
1993
+ return cands[0]
1994
+ if fid:
1995
+ same_file = [c for c in cands if self.file_of.get(c) == fid]
1996
+ if same_file:
1997
+ return same_file[0]
1998
+ if lang == "csharp":
1999
+ here = set(self.cs_usings[fid])
2000
+ for ns in self.results[fid].declares: # a namespace sees its parents
2001
+ parts = ns.split(".")
2002
+ here.update(".".join(parts[:i]) for i in range(1, len(parts) + 1))
2003
+ here.add("")
2004
+ visible = [c for c in cands if self.nodes[c].attrs.get("namespace") in here]
2005
+ if len(visible) >= 1:
2006
+ return visible[0]
2007
+ return None
2008
+
2009
+ def _resolve_types(self) -> None:
2010
+ seen = set()
2011
+ for fid, res in self.results.items():
2012
+ lang = self.file_lang[fid]
2013
+ adapter = BY_LANGUAGE[lang].NAME
2014
+ for ref in res.type_refs:
2015
+ for i, name in enumerate(ref.names):
2016
+ tid = self._type(lang, name, ref.src_id)
2017
+ if tid is None:
2018
+ self.stats[adapter]["type_refs_external"] += 1
2019
+ continue
2020
+ self.stats[adapter]["type_refs_resolved"] += 1
2021
+ if ref.role == "base":
2022
+ if i > 0:
2023
+ kind, attrs = "uses_type", {"role": "generic_arg"}
2024
+ else:
2025
+ target_iface = self.nodes[tid].attrs.get("native_kind") == "interface"
2026
+ src_iface = self.nodes[ref.src_id].attrs.get("native_kind") == "interface"
2027
+ kind, attrs = ("implements" if target_iface and not src_iface else "extends"), {}
2028
+ self.bases[ref.src_id].append(tid)
2029
+ elif ref.role == "instantiate":
2030
+ kind, attrs = ("instantiates" if i == 0 else "uses_type"), ({} if i == 0 else {"role": "generic_arg"})
2031
+ else:
2032
+ kind, attrs = "uses_type", {"role": ref.role if i == 0 else "generic_arg"}
2033
+ key = (kind, ref.src_id, tid, attrs.get("role"))
2034
+ if key in seen or ref.src_id == tid:
2035
+ continue
2036
+ seen.add(key)
2037
+ self.edges.append(Edge(kind, ref.src_id, tid, "heuristic", attrs))
2038
+
2039
+ # -- calls ---------------------------------------------------------------
2040
+ def _chain(self, type_id: Optional[str], enclosing: bool = True) -> list[str]:
2041
+ """A type, its bases, and (unless `enclosing` is off) its enclosing types, nearest first. A name written bare
2042
+ inside a nested type can mean the enclosing type's static member; a call on a receiver (`this`, `self`, a
2043
+ typed variable) reaches only the receiver's own type and its bases."""
2044
+ out, queue = [], [type_id] if type_id else []
2045
+ while queue:
2046
+ t = queue.pop(0)
2047
+ if t in out or t not in self.nodes:
2048
+ continue
2049
+ out.append(t)
2050
+ queue.extend(self.bases.get(t, []))
2051
+ parent = self.nodes[t].parent_id
2052
+ if enclosing and parent in self.nodes and self.nodes[parent].kind == "type":
2053
+ queue.append(parent)
2054
+ return out
2055
+
2056
+ def _pick(self, cands: list[Node], argc: int, skip: int = 0) -> list[Node]:
2057
+ """Overloads that fit a call. `skip` is 1 for an extension method, whose first parameter is the receiver."""
2058
+ if argc < 0:
2059
+ return list(cands) # a function named, not called: nothing to match arguments against
2060
+ argc += skip
2061
+ fit = [c for c in cands if c.attrs.get("argc_min", 0) <= argc <= c.attrs.get("argc_max", 99)]
2062
+ call = getattr(self, "_call", None)
2063
+ if len(fit) > 1 and call is not None:
2064
+ # Several overloads take this many arguments. Narrow by what the call site shows.
2065
+ if call.targs:
2066
+ fit = [c for c in fit if c.attrs.get("generic_arity") == call.targs] or fit
2067
+ hints = call.args
2068
+
2069
+ def possible(c: Node) -> bool:
2070
+ want = c.attrs.get("delegate_arity") or []
2071
+ for i, h in enumerate(hints, skip):
2072
+ if isinstance(h, int) and not isinstance(h, bool) and i < len(want):
2073
+ if want[i] == -2 or (want[i] >= 0 and want[i] != h):
2074
+ return False # a lambda passed where no delegate of that shape is taken
2075
+ elif isinstance(h, str) and i < len(want) and want[i] >= 0 and h in ("int", "double", "bool", "char", "string"):
2076
+ return False # a literal passed where a delegate is taken
2077
+ return True
2078
+
2079
+ def score(c: Node) -> int:
2080
+ want, types = c.attrs.get("delegate_arity") or [], c.attrs.get("param_types") or []
2081
+ n = 0
2082
+ for i, h in enumerate(hints, skip):
2083
+ if isinstance(h, int) and i < len(want) and want[i] == h:
2084
+ n += 1
2085
+ elif isinstance(h, str) and i < len(types) and types[i].split("`")[0] == h:
2086
+ n += 1
2087
+ return n
2088
+ fit = [c for c in fit if possible(c)] or fit
2089
+ best = max(score(c) for c in fit)
2090
+ fit = [c for c in fit if score(c) == best]
2091
+ return fit or cands
2092
+
2093
+ def _methods(self, type_id: Optional[str], name: str, argc: int, enclosing: bool = True) -> list[Node]:
2094
+ for t in self._chain(type_id, enclosing):
2095
+ cands = self.members.get(t, {}).get(name)
2096
+ if cands:
2097
+ return self._pick(cands, argc)
2098
+ return []
2099
+
2100
+ def _receiver_type(self, lang: str, call: CallSite) -> tuple[Optional[str], bool]:
2101
+ """Returns (type id, known). known=True with None means an external type."""
2102
+ r = call.receiver
2103
+ if r in (None, "this"):
2104
+ return call.enclosing_type, call.enclosing_type is not None
2105
+ if r == "base":
2106
+ bases = self.bases.get(call.enclosing_type or "", [])
2107
+ return (bases[0] if bases else None), True
2108
+ if call.receiver_type:
2109
+ return self._type(lang, call.receiver_type, call.src_id), True
2110
+ if call.chain is not None and not call.receiver_type:
2111
+ got = self._returned_type(lang, call.chain)
2112
+ if got is not None:
2113
+ return got
2114
+ if r in ("?", "[]"):
2115
+ return None, False
2116
+ if r in getattr(BY_LANGUAGE[lang], "GLOBALS", ()) and not self._type(lang, r, call.src_id):
2117
+ return None, True # JSON.stringify, Math.max: the platform's, not ours
2118
+ if r.startswith("."):
2119
+ names = self.field_types_global.get((lang, r[1:]), set())
2120
+ ids = {self._type(lang, n, call.src_id) for n in names}
2121
+ ids.discard(None)
2122
+ if len(ids) == 1:
2123
+ return ids.pop(), True
2124
+ if names and not ids:
2125
+ return None, True # every field of that name has an external type
2126
+ return None, False
2127
+ if lang == "python":
2128
+ # A parameter filled by a pytest fixture, seen from the test or a function nested in it.
2129
+ cur, shadowed = call.src_id, False
2130
+ while cur in self.nodes and self.nodes[cur].kind == "callable":
2131
+ if (cur, r) in self.py_param_type:
2132
+ return self.py_param_type[(cur, r)], True
2133
+ if r in (self.nodes[cur].attrs.get("params") or []):
2134
+ shadowed = True
2135
+ break
2136
+ cur = self.nodes[cur].parent_id
2137
+ fid = self.file_of.get(call.src_id)
2138
+ if not shadowed and call.chain is None and fid and (r in self.py_vars.get(fid, {}) or self._py_bound(fid, call.src_id, r)):
2139
+ tid = self._py_var(fid, r) # a module-level variable, of this file or imported
2140
+ if tid:
2141
+ return tid, True
2142
+ # A bare identifier: a field of the enclosing type chain, or a type name (static call).
2143
+ for t in self._chain(call.enclosing_type):
2144
+ tn = self.field_type.get(t, {}).get(r)
2145
+ if tn:
2146
+ return self._type(lang, tn, call.src_id), True
2147
+ tid = self._type(lang, r, call.src_id)
2148
+ if tid:
2149
+ return tid, True
2150
+ if lang == "csharp" and r[:1].isupper() and not any(
2151
+ r in self.field_names.get(t, ()) for t in self._chain(call.enclosing_type)):
2152
+ return None, True # PascalCase, not a member, not ours: a type from outside (File.Open)
2153
+ return None, False
2154
+
2155
+ WRAPPERS = {"Task", "ValueTask", "Task`1", "ValueTask`1", "Nullable`1"}
2156
+
2157
+ def _returned_type(self, lang: str, inner: CallSite, depth: int = 0) -> Optional[tuple]:
2158
+ """The type of a call's result, from the declared return type of what it resolves to.
2159
+ Returns (type id or None, known) like _receiver_type, or None when nothing can be said."""
2160
+ key = self._ckey(inner)
2161
+ if key in self._chain_memo:
2162
+ return self._chain_memo[key]
2163
+ self._chain_memo[key] = None
2164
+ if depth > 6:
2165
+ return None
2166
+ if inner.attr:
2167
+ out = self._attr_type(lang, inner)
2168
+ self._chain_memo[key] = out
2169
+ return out
2170
+ saved = (getattr(self, "_guessed", False), getattr(self, "_call", None), getattr(self, "_last_src", None))
2171
+ self._guessed = False
2172
+ stats_before = {k: dict(v) for k, v in self.stats.items()}
2173
+ self._call = inner
2174
+ fid = self.file_of.get(inner.src_id)
2175
+ targets = self._resolve_call(lang, fid, inner) if fid else None
2176
+ guessed = self._guessed
2177
+ self._guessed, self._call, self._last_src = saved[0], saved[1], saved[2]
2178
+ for k, v in stats_before.items(): # the inner call is counted when its own turn comes
2179
+ self.stats[k].clear()
2180
+ self.stats[k].update(v)
2181
+ out = None
2182
+ if targets and not guessed:
2183
+ t = targets[0]
2184
+ if t.name in CTORS:
2185
+ out = (t.parent_id, True)
2186
+ elif lang == "python" and t.attrs.get("returns_fn") and f"{t.id}.{t.attrs['returns_fn']}" in self.nodes:
2187
+ out = (f"{t.id}.{t.attrs['returns_fn']}", True) # a function value: calling it runs that function
2188
+ elif lang in FILE_MODULE:
2189
+ if t.attrs.get("returns"):
2190
+ tid = self._type(lang, t.attrs["returns"], t.id)
2191
+ out = (tid, True) if tid else None
2192
+ else:
2193
+ names = [n for n in t.attrs.get("returns_names") or [] if n not in self.WRAPPERS]
2194
+ if names and names[0] not in (t.attrs.get("type_params") or []):
2195
+ tid = self._type(lang, names[0], t.id)
2196
+ owner_params = re.findall(r"[A-Za-z_]\w*", (self.nodes[t.parent_id].attrs.get("signature") or "").split(":")[0].partition("<")[2]) \
2197
+ if t.parent_id in self.nodes else []
2198
+ if tid:
2199
+ out = (tid, True)
2200
+ elif names[0].split("`")[0] not in owner_params and names[0] not in ("void", "var", "dynamic", "object"):
2201
+ out = (None, True) # a declared type from outside the workspace
2202
+ self._chain_memo[key] = out
2203
+ return out
2204
+
2205
+ def _extensions(self, lang: str, fid: str, tid: str, name: str, argc: int) -> list[Node]:
2206
+ """Extension methods called on a receiver of a known type: static methods whose `this` parameter is that type."""
2207
+ cands = [c for c in self.by_name.get((lang, name), ()) if c.attrs.get("is_extension") and self._can_see(fid, c.id)]
2208
+ if not cands:
2209
+ return []
2210
+ # Foo and Foo<T> are different receivers: compare the name together with its generic arity.
2211
+ names = {self.nodes[t].name.split("`")[0] + (f"`{_arity(t)}" if _arity(t) else "")
2212
+ for t in self._chain(tid) if t in self.nodes}
2213
+ exact = [c for c in cands if (c.attrs.get("param_types") or [""])[0] in names]
2214
+ loose = [c for c in cands if (c.attrs.get("param_types") or [""])[0] in (c.attrs.get("type_params") or [])]
2215
+ pool = exact or loose
2216
+ if not pool:
2217
+ return []
2218
+ if not exact:
2219
+ self._guessed = True # `this TBuilder builder`: the constraint is not checked
2220
+ return self._pick(pool, argc, skip=1)
2221
+
2222
+ def _resolve_calls(self) -> None:
2223
+ self._chain_memo: dict = {}
2224
+ self._call = None
2225
+ self._arg_log: list = [] # Python: (function, a call that reaches it), for _py_argument_flow
2226
+ self._pending: list = [] # Python: calls nothing resolved, for _py_argument_flow
2227
+ for (tid, fname), method in self.field_calls.items():
2228
+ # self.config = self.make_config(): the field holds what that method is declared to return.
2229
+ if fname in self.field_type.get(tid, {}):
2230
+ continue
2231
+ for m in self._methods(tid, method, -1):
2232
+ if m.attrs.get("returns"):
2233
+ self.field_type[tid][fname] = m.attrs["returns"]
2234
+ self.field_types_global[(m.language, fname)].add(m.attrs["returns"])
2235
+ break
2236
+ # Names seen on receivers of a known outside type (List.Add, dict.get). A call to such a
2237
+ # name on a receiver of unknown type is never guessed.
2238
+ self.outside_names: set = set()
2239
+ for a in BY_LANGUAGE.values():
2240
+ self.outside_names.update((a.LANGUAGE, n) for n in getattr(a, "COMMON_METHODS", ()))
2241
+ self._py_fixtures()
2242
+ for fid, res in self._calls_pass(1):
2243
+ lang = self.file_lang[fid]
2244
+ for call in self._open(fid, res).calls:
2245
+ if call.receiver not in (None, "this", "base") and call.name != ".ctor":
2246
+ if lang in FILE_MODULE and (self._py_bound(fid, call.src_id, call.receiver) if lang == "python" else call.receiver in self.py_names[fid]):
2247
+ continue
2248
+ if call.ref:
2249
+ continue
2250
+ tid, known = self._receiver_type(lang, call)
2251
+ if known and tid is None:
2252
+ self.outside_names.add((lang, call.name))
2253
+ for fid, res in self._calls_pass(2):
2254
+ lang = self.file_lang[fid]
2255
+ adapter = BY_LANGUAGE[lang].NAME
2256
+ st = self.stats[adapter]
2257
+ for call in self._open(fid, res).calls:
2258
+ if call.ref:
2259
+ # A function handed over by name. Linked when the name is one of ours; otherwise it was a value.
2260
+ self._guessed, self._call = False, None
2261
+ before = dict(st)
2262
+ targets = [t for t in (self._resolve_call(lang, fid, call) or []) if t.kind == "callable"]
2263
+ st.clear()
2264
+ st.update(before)
2265
+ if targets and not self._guessed:
2266
+ st["references_linked"] += 1
2267
+ for t in targets:
2268
+ self.calls.append((call.src_id, t.id, "reference", "heuristic", call.line))
2269
+ self.call_col[(call.src_id, t.id, call.line)] = call.col
2270
+ continue
2271
+ st["calls_total"] += 1
2272
+ self._guessed = False
2273
+ self._call = call
2274
+ targets = self._resolve_call(lang, fid, call)
2275
+ if targets is None:
2276
+ st["calls_external"] += 1
2277
+ elif not targets:
2278
+ st["calls_unresolved"] += 1
2279
+ else:
2280
+ st["calls_resolved"] += 1
2281
+ for t in targets:
2282
+ dispatch = "virtual" if t.attrs.get("is_virtual") and lang == "csharp" else "static"
2283
+ self.calls.append((call.src_id, t.id, dispatch,
2284
+ "guess" if self._guessed else "heuristic", call.line))
2285
+ self.call_col[(call.src_id, t.id, call.line)] = call.col
2286
+ if lang == "python":
2287
+ if targets and not self._guessed:
2288
+ for t in targets:
2289
+ self._arg_log.append((t.id, call))
2290
+ elif not targets:
2291
+ self._pending.append(call)
2292
+ self._shut()
2293
+ # Kept as lists on the indexer, so an incremental run records each file's share with its turn and replays it.
2294
+ incoming: dict[str, list] = defaultdict(list) # Python: function -> the calls that reach it
2295
+ for tid, call in self._arg_log:
2296
+ incoming[tid].append(call)
2297
+ self._py_argument_flow(incoming, self._pending)
2298
+
2299
+ def _py_argument_flow(self, incoming: dict, pending: list) -> None:
2300
+ """Calls made on a value handed in from outside: `app(environ, start)` on a parameter, `self.repo.save()` on
2301
+ a field set from one. The value's class is whatever the callers pass, followed back through parameters,
2302
+ fields set from parameters and `*args` passed on. Linked to that class's method (`__call__` when the value
2303
+ itself is called), as a guess, since a caller the map does not see may pass something else."""
2304
+ if not pending:
2305
+ return
2306
+ memo: dict = {}
2307
+ active: set = set()
2308
+ cut = [0, 0] # answers left incomplete by a cycle or a limit; questions asked for the current call
2309
+
2310
+ def param_owner(src: str, name: str) -> Optional[tuple]:
2311
+ cur = src
2312
+ while cur in self.nodes and self.nodes[cur].kind == "callable":
2313
+ params = self.nodes[cur].attrs.get("params") or []
2314
+ if name in params:
2315
+ return cur, params.index(name)
2316
+ cur = self.nodes[cur].parent_id
2317
+ return None
2318
+
2319
+ def narrowed(src: str, owner: str, name: str) -> bool:
2320
+ """Is the parameter tested with isinstance() between the call and the function that takes it?"""
2321
+ cur = src
2322
+ while cur in self.nodes and self.nodes[cur].kind == "callable":
2323
+ if name in (self.nodes[cur].attrs.get("narrowed") or ()):
2324
+ return True
2325
+ if cur == owner:
2326
+ return False
2327
+ cur = self.nodes[cur].parent_id
2328
+ return False
2329
+
2330
+ def field_types(type_id: Optional[str], name: str, depth: int) -> set:
2331
+ for t in self._chain(type_id):
2332
+ f = self.nodes.get(f"{t}.{name}")
2333
+ if f is None or f.kind != "field":
2334
+ continue
2335
+ tn = self.field_type.get(t, {}).get(name)
2336
+ tid = self._type("python", tn, f.id) if tn else None
2337
+ if tid:
2338
+ return {tid}
2339
+ out = set()
2340
+ for fn, pname in f.attrs.get("from_params") or []: # self.app = app, in the constructor
2341
+ owner = param_owner(fn, pname)
2342
+ out |= arg_types(owner[0], owner[1], depth + 1) if owner else set()
2343
+ return out
2344
+ return set()
2345
+
2346
+ def arg_types(fn_id: str, pos: int, depth: int = 0) -> set:
2347
+ """Classes of ours passed as the pos-th positional argument of fn_id."""
2348
+ key = (fn_id, pos)
2349
+ if key in memo:
2350
+ return memo[key]
2351
+ fn = self.nodes.get(fn_id)
2352
+ cut[1] += 1
2353
+ if fn is None or key in active or depth > 8 or cut[1] > 5000:
2354
+ cut[0] += 1 # not cached: the same question asked from higher up may get further
2355
+ return set()
2356
+ params, star = fn.attrs.get("params") or [], fn.attrs.get("star_at")
2357
+ if pos < len(params) and (star is None or pos < star):
2358
+ declared = (fn.attrs.get("ptypes") or {}).get(params[pos])
2359
+ tid = self._type("python", declared, fn_id) if declared else None
2360
+ if tid:
2361
+ memo[key] = {tid}
2362
+ return memo[key]
2363
+ active.add(key)
2364
+ before, out = cut[0], set()
2365
+ for c in incoming.get(fn_id, ()):
2366
+ hints = c.args or ()
2367
+ spread = next((k for k, h in enumerate(hints[:pos + 1]) if h == "@*"), None)
2368
+ if spread is not None:
2369
+ caller = self.nodes.get(c.src_id)
2370
+ if caller is not None and caller.attrs.get("star_at") is not None:
2371
+ out |= arg_types(c.src_id, caller.attrs["star_at"] + pos - spread, depth + 1)
2372
+ continue
2373
+ h = hints[pos] if pos < len(hints) else None
2374
+ if h is None:
2375
+ continue
2376
+ if h == "@self":
2377
+ out |= {c.enclosing_type} if c.enclosing_type else set()
2378
+ elif h.startswith("@p:"):
2379
+ owner = param_owner(c.src_id, h[3:])
2380
+ out |= arg_types(owner[0], owner[1], depth + 1) if owner else set()
2381
+ elif h.startswith("@f:"):
2382
+ out |= field_types(c.enclosing_type, h[3:], depth)
2383
+ else:
2384
+ tid = self._type("python", h, c.src_id)
2385
+ out |= {tid} if tid else set()
2386
+ active.discard(key)
2387
+ if cut[0] == before:
2388
+ memo[key] = out
2389
+ return out
2390
+
2391
+ st = self.stats[BY_LANGUAGE["python"].NAME]
2392
+ for call in pending:
2393
+ r, method = call.receiver, call.name
2394
+ types: set = set()
2395
+ cut[1] = 0
2396
+ if r is None or r not in ("this", "base", "?"):
2397
+ owner = param_owner(call.src_id, r or call.name)
2398
+ if owner is not None and narrowed(call.src_id, owner[0], r or call.name):
2399
+ owner = None
2400
+ if owner is not None and (r is None or call.receiver_type is None or not self._type("python", call.receiver_type, call.src_id)):
2401
+ types = arg_types(*owner)
2402
+ method = "__call__" if r is None else method
2403
+ elif r == "this" and call.enclosing_type and call.name in self.field_names.get(call.enclosing_type, ()):
2404
+ types, method = field_types(call.enclosing_type, call.name, 0), "__call__" # self.handler(x)
2405
+ if r is not None and r.startswith(".") and call.enclosing_type:
2406
+ types = field_types(call.enclosing_type, r[1:], 0)
2407
+ for tid in sorted(types):
2408
+ self._call = call
2409
+ for t in self._methods(tid, method, call.argc):
2410
+ st["calls_by_argument_flow"] += 1
2411
+ self.calls.append((call.src_id, t.id, "static", "guess", call.line))
2412
+ self.call_col[(call.src_id, t.id, call.line)] = call.col
2413
+
2414
+ def _calls_pass(self, phase: int):
2415
+ """The files whose calls _resolve_calls goes through, in its first pass (names seen on outside types) and
2416
+ its second (the calls themselves). Every file, in order; leyline.incremental narrows it to the files an
2417
+ edit can have changed and fills in the rest from the last run."""
2418
+ return self.results.items()
2419
+
2420
+ _DECL = [ # (pattern, group of the name, group of the type): how typed languages write a variable's type
2421
+ (re.compile(r"\b([A-Z]\w*)(?:<[^<>;=()]*>)?(?:\[\])?\??\s*[*&]?\s+(\w+)\s*(?=[=;,)]|$)"), 2, 1), # Foo x (C#, Java, C++)
2422
+ (re.compile(r"\b(\w+)\s*:\s*&?(?:mut\s+|readonly\s+)?\(?(?:[a-z_]\w*(?:::|\.))*([A-Z]\w*)"), 1, 2), # x: Foo, x: &mod::Foo (TS, Kotlin, Swift, Rust, Python)
2423
+ (re.compile(r"\b(\w+)\s*(?::=|=)\s*(?:new\s+|&|await\s+)?([A-Z]\w*)(?:<[^<>]*>)?(?:::new)?\s*[({]"), 1, 2), # x = new Foo( / Foo{ / Foo::new(
2424
+ (re.compile(r"\b(\w+)\s+\*?(?:[a-z_]\w*\.)?([A-Z]\w*)\s*[,)]"), 1, 2), # (x *Foo, s *store.Store) (Go parameters)
2425
+ (re.compile(r"\bvar\s+(\w+)\s+\*?(?:[a-z_]\w*\.)?([A-Z]\w*)"), 1, 2), # var x Foo, var c store.Cache (Go)
2426
+ # built-in types, which tell an overload apart: int x, boolean b[], x int, err error
2427
+ (re.compile(r"\b(byte|short|int|long|float|double|boolean|bool|char|string|str|rune)(?:\[\])?\s+(\w+)\s*(?=[=;,)])"), 2, 1),
2428
+ (re.compile(r"\b(\w+)\s+(byte|int|int32|int64|uint|float32|float64|bool|string|rune|error)\s*[,)]"), 1, 2),
2429
+ ]
2430
+
2431
+ def _generic_var_type(self, lang: str, src_id: str, var: str, enclosing_type: Optional[str]) -> Optional[str]:
2432
+ """The type a variable or field is declared with, read from the text of the function and of its type."""
2433
+ if not re.fullmatch(r"[A-Za-z_]\w*", var):
2434
+ return None
2435
+ for holder in (src_id, enclosing_type):
2436
+ if not holder or holder not in self.nodes:
2437
+ continue
2438
+ decls = self._generic_decls(lang, holder)[0]
2439
+ if var in decls:
2440
+ if decls[var] is None:
2441
+ return "external" # declared with a type that is not in the repo
2442
+ return self._type(lang, decls[var], src_id) or (self.types_by_name.get((lang, decls[var])) or [None])[0]
2443
+ return None
2444
+
2445
+ def _generic_decls(self, lang: str, holder: str) -> tuple:
2446
+ """({variable: repo type name, or None for a type from outside}, {variable: type name as written})."""
2447
+ got = self._decl_cache.get(holder)
2448
+ if got is None:
2449
+ n = self.nodes[holder]
2450
+ fid = self.file_of.get(holder)
2451
+ read = self._decls_read(fid).get(holder)
2452
+ if read is not None and read[:2] == (n.span_start, n.span_end):
2453
+ found = read[2] # read by the parse worker from the same text
2454
+ else:
2455
+ found = _declared(self._file_text(fid) if fid else [], n.span_start, n.span_end)
2456
+ decls, raw = {}, {}
2457
+ for name, tnames in found.items():
2458
+ raw[name] = tnames[0]
2459
+ typed = [t for t in tnames if t not in NOT_TYPES]
2460
+ if typed:
2461
+ # the first type written for it, or, if that one is not in the repo, the first after it that is
2462
+ decls[name] = next((t for t in typed if self.types_by_name.get((lang, t))), None)
2463
+ if len(self._decl_cache) >= 4096:
2464
+ # Asked about the functions of the file being resolved; made again the same when asked later.
2465
+ # Kept for every function, it was hundreds of MB on a large repository.
2466
+ self._decl_cache.clear()
2467
+ got = self._decl_cache[holder] = (decls, raw)
2468
+ return got
2469
+
2470
+ _INTS = re.compile(r"(?i)^(int|long|short|byte|integer|bigint|biginteger|[iu](8|16|32|64|128|size)|u?int(8|16|32|64)?|rune)$")
2471
+ _FLOATS = re.compile(r"(?i)^(float|double|f32|f64|float32|float64|decimal|bigdecimal|number)$")
2472
+ _STRS = re.compile(r"(?i)^(string|str|charsequence)$")
2473
+ _BOOLS = re.compile(r"(?i)^(bool|boolean)$")
2474
+ _PRIMS = {"int", "long", "short", "byte", "float", "double", "boolean", "char", "bool"}
2475
+ _ANY = {"Object", "object", "any", "Any", "AnyObject", "dynamic", "interface", "unknown"}
2476
+
2477
+ def _supers(self, tid: str) -> set:
2478
+ out, queue = set(), [tid]
2479
+ while queue:
2480
+ t = queue.pop()
2481
+ if t not in out:
2482
+ out.add(t)
2483
+ queue.extend(self.bases.get(t, []))
2484
+ return out
2485
+
2486
+ def _arg_fits(self, lang: str, hint: str, ptype: str) -> int:
2487
+ """How well an argument fits a declared parameter type: 2 the same, 1 compatible, 0 cannot tell, -1 cannot be."""
2488
+ p = ptype.rstrip(".")
2489
+ if not p or not hint or hint == "fn":
2490
+ return 0
2491
+ repo_p = self.types_by_name.get((lang, p))
2492
+ kinds = {"string": self._STRS, "int": self._INTS, "float": self._FLOATS, "bool": self._BOOLS}
2493
+ boxed, named = False, hint
2494
+ if hint not in kinds and hint not in ("char", "null") and not self.types_by_name.get((lang, hint)):
2495
+ # a built-in type by name (int, Integer, String): the same as the literal of that kind
2496
+ kind = "char" if hint.lower() in ("char", "character") else \
2497
+ next((k for k, pat in kinds.items() if pat.match(hint) and hint.lower() not in ("number", "decimal")), None)
2498
+ if kind:
2499
+ boxed, hint = hint[:1].isupper() and kind != "string", kind
2500
+ if hint in kinds or hint == "char":
2501
+ # a literal is the built-in type exactly (int, bool); its boxed or wider form takes it too
2502
+ exact = 2 if p[:1].islower() != boxed or hint == "string" else 1
2503
+ if hint == "char" and p.lower() in ("char", "character", "rune"):
2504
+ return exact
2505
+ if hint in kinds and kinds[hint].match(p):
2506
+ if named.lower() == p.lower() and named != hint:
2507
+ return 3 # the very type named: Float.NaN to float rather than double
2508
+ if named == "float" and re.fullmatch(r"(?i)float|f32|float32", p):
2509
+ return 1 # a decimal literal is a double unless marked otherwise
2510
+ return exact if p.lower() not in ("number", "decimal", "bigdecimal", "biginteger", "bigint") else 1
2511
+ if hint in ("int", "char") and self._FLOATS.match(p) or hint == "char" and self._INTS.match(p):
2512
+ return 1 # widened
2513
+ other = any(k.match(p) for k in kinds.values()) or p in self._PRIMS
2514
+ return -1 if other or repo_p else 0
2515
+ if hint == "null":
2516
+ return -1 if p in self._PRIMS else 0
2517
+ if any(k.match(p) for k in kinds.values()) or p in self._PRIMS:
2518
+ return -1 # a type that is not a built-in one passed where a built-in one is taken
2519
+ if hint == p:
2520
+ return 2
2521
+ repo_h = self.types_by_name.get((lang, hint))
2522
+ if repo_h and repo_p:
2523
+ return 1 if any(t in self._supers(h) for h in repo_h for t in repo_p) else -1
2524
+ if repo_h and (p in self._PRIMS or any(k.match(p) for k in kinds.values())):
2525
+ return -1
2526
+ if repo_p and not repo_h and (len(hint) > 2 or not hint.isupper()):
2527
+ return -1 # a type from outside cannot be one of ours (a one-letter name is a type parameter)
2528
+ if repo_h and not repo_p and p not in self._ANY and (len(p) > 2 or not p.isupper()):
2529
+ # one of ours passed where a type from outside is taken: only if it says it extends that type
2530
+ said = " ".join(self.nodes[t].attrs.get("signature") or "" for h in repo_h for t in self._supers(h) if t in self.nodes)
2531
+ return 1 if re.search(rf"\b{re.escape(p)}\b", said) else -1
2532
+ return 0
2533
+
2534
+ def _generic_returns(self, lang: str, inner: CallSite) -> Optional[tuple]:
2535
+ """_returned_type for the generic resolver, without the edges resolving the inner call would record twice."""
2536
+ if inner.receiver is None and self.types_by_name.get((lang, inner.name)):
2537
+ return (self.types_by_name[(lang, inner.name)][0], True) # Foo(): a constructor called by the type's name
2538
+ n_edges = len(self.edges)
2539
+ out = self._returned_type(lang, inner)
2540
+ del self.edges[n_edges:]
2541
+ return out
2542
+
2543
+ def _generic_return_name(self, lang: str, inner: CallSite) -> Optional[str]:
2544
+ """The type name a call's target declares it returns, from outside the repo or not (None when the call
2545
+ does not resolve, or when what it returns is a type parameter)."""
2546
+ key = ("ret", self._ckey(inner))
2547
+ if key in self._chain_memo:
2548
+ return self._chain_memo[key]
2549
+ self._chain_memo[key] = None
2550
+ if inner.receiver is None and self.types_by_name.get((lang, inner.name)):
2551
+ out = inner.name
2552
+ else:
2553
+ saved = (getattr(self, "_guessed", False), getattr(self, "_call", None))
2554
+ stats_before = {k: dict(v) for k, v in self.stats.items()}
2555
+ n_edges = len(self.edges)
2556
+ self._guessed = False
2557
+ fid = self.file_of.get(inner.src_id)
2558
+ targets = self._resolve_call(lang, fid, inner) if fid else None
2559
+ guessed = self._guessed
2560
+ self._guessed, self._call = saved
2561
+ for k, v in stats_before.items():
2562
+ self.stats[k].clear()
2563
+ self.stats[k].update(v)
2564
+ del self.edges[n_edges:]
2565
+ names = {(t.attrs.get("returns_names") or [None])[0] for t in targets or ()
2566
+ if (t.attrs.get("returns_names") or [None])[0] not in (t.attrs.get("type_params") or ())}
2567
+ out = names.pop() if targets and not guessed and len(names) == 1 else None
2568
+ self._chain_memo[key] = out
2569
+ return out
2570
+
2571
+ def _hint_type(self, lang: str, call: CallSite, h) -> Optional[str]:
2572
+ """The type name an argument hint stands for: `$x` a variable, `$a.b` a field, a call its return type."""
2573
+ if isinstance(h, CallSite):
2574
+ return self._generic_return_name(lang, h)
2575
+ if not isinstance(h, str) or not h.startswith("$"):
2576
+ return h if isinstance(h, str) else None
2577
+ var, _, field = h[1:].partition(".")
2578
+ if field:
2579
+ if not self.types_by_name.get((lang, var)) and any(k.match(var) for k in (self._INTS, self._FLOATS, self._BOOLS)):
2580
+ return var # Integer.MAX_VALUE, Double.NaN: a constant of that built-in type
2581
+ if var in ("this", "self"):
2582
+ tids = [call.enclosing_type] if call.enclosing_type else []
2583
+ else:
2584
+ tids = self.types_by_name.get((lang, var)) or []
2585
+ if not tids:
2586
+ tid = self._generic_var_type(lang, call.src_id, var, call.enclosing_type)
2587
+ tids = [tid] if tid and tid != "external" else []
2588
+ for tid in tids:
2589
+ for t in self._chain(tid):
2590
+ if field in self.field_type.get(t, {}):
2591
+ return self.field_type[t][field]
2592
+ return None
2593
+ for holder in (call.src_id, call.enclosing_type):
2594
+ if holder and holder in self.nodes:
2595
+ raw = self._generic_decls(lang, holder)[1]
2596
+ if var in raw:
2597
+ return raw[var]
2598
+ return None
2599
+
2600
+ def _by_arg_types(self, lang: str, call: CallSite, fns: list) -> list:
2601
+ """Overloads with as many parameters: keep those whose parameter types fit what the arguments show."""
2602
+ hints = [self._hint_type(lang, call, h) for h in call.args]
2603
+ if not any(hints):
2604
+ return fns
2605
+ scored = []
2606
+ for f in fns:
2607
+ types, score = f.attrs.get("param_types") or [], 0
2608
+ for i, h in enumerate(hints):
2609
+ p = types[i] if i < len(types) else (types[-1] if types and types[-1].endswith("...") else "")
2610
+ v = self._arg_fits(lang, h, p) if h else 0
2611
+ if v < 0:
2612
+ score = -1
2613
+ break
2614
+ score += v
2615
+ scored.append((score, f))
2616
+ best = max(sc for sc, _ in scored)
2617
+ return [f for sc, f in scored if sc == best] if best >= 0 else fns
2618
+
2619
+ def _file_text(self, fid: str) -> list[str]:
2620
+ if fid not in self._text_cache:
2621
+ if len(self._text_cache) >= 8:
2622
+ # Calls are resolved a file at a time, so a few files are enough; keeping every file's text
2623
+ # cost hundreds of MB on a large repo.
2624
+ self._text_cache.clear()
2625
+ try:
2626
+ self._text_cache[fid] = source_lines(self.repos[_repo_of(fid)] / self.nodes[fid].path)
2627
+ except OSError:
2628
+ self._text_cache[fid] = []
2629
+ return self._text_cache[fid]
2630
+
2631
+ def _near(self, fid: str) -> set:
2632
+ """Files this one imports, and the files those import in turn (a package's index re-exporting its parts)."""
2633
+ key = ("near", fid)
2634
+ got = self._file_set(key)
2635
+ if got is None:
2636
+ first = set(self.import_targets.get(fid, ()))
2637
+ got = self._keep_set(key, first | {t for f in first for t in self.import_targets.get(f, ())})
2638
+ return got
2639
+
2640
+ def _name_index(self, lang: str, name: str) -> dict:
2641
+ """Where the callables of one name are declared, looked up by file and directory. A common name (get, run)
2642
+ has thousands of declarations in a large repo, and scanning them for every call was quadratic."""
2643
+ key = (lang, name)
2644
+ ix = self._name_ix.get(key)
2645
+ if ix is None:
2646
+ ix = {"free": [], "typed": [], "free_file": defaultdict(list), "typed_file": defaultdict(list),
2647
+ "free_dir": {}, "free_dir_base": {}, "stem": {}, "stem_free": {}, "owners": set(), "typed_owners": set()}
2648
+ for i, c in enumerate(self.by_name.get(key, ())):
2649
+ f = self.file_of.get(c.id)
2650
+ typed = bool(c.attrs.get("type_id"))
2651
+ ix["owners"].add(c.parent_id)
2652
+ if typed:
2653
+ ix["typed"].append(c)
2654
+ ix["typed_file"][f].append((i, c))
2655
+ ix["typed_owners"].add(c.parent_id)
2656
+ else:
2657
+ ix["free"].append(c)
2658
+ ix["free_file"][f].append((i, c))
2659
+ if f is None:
2660
+ continue
2661
+ d = self.nodes[f].path.rpartition("/")[0]
2662
+ if not typed:
2663
+ ix["free_dir"].setdefault(d, c)
2664
+ ix["free_dir_base"].setdefault(d.rsplit("/", 1)[-1], c)
2665
+ ix["stem"].setdefault(self.nodes[f].name.rsplit(".", 1)[0], c)
2666
+ if not typed:
2667
+ ix["stem_free"].setdefault(self.nodes[f].name.rsplit(".", 1)[0], c)
2668
+ self._name_ix[key] = ix
2669
+ return ix
2670
+
2671
+ @staticmethod
2672
+ def _in_files(by_file: dict, files, also: Optional[str] = None) -> list:
2673
+ """The declarations in any of these files (or in `also`), in declaration order. Walks whichever side is
2674
+ smaller: a Go package's files all import each other, so `files` can be thousands long for every call."""
2675
+ if len(by_file) < len(files):
2676
+ hits = [ic for f, ics in by_file.items() if f in files or f == also for ic in ics]
2677
+ else:
2678
+ hits = [ic for f in files for ic in by_file.get(f, ())]
2679
+ if also is not None and also not in files:
2680
+ hits.extend(by_file.get(also, ()))
2681
+ hits.sort(key=lambda ic: ic[0])
2682
+ return [c for _, c in hits]
2683
+
2684
+ def _generic_call(self, lang: str, fid: str, call: CallSite) -> Optional[list[Node]]:
2685
+ """A call read by the generic adapter: only the name and what the text shows of the receiver are known.
2686
+ Nearest first: the caller's own type, its file, its directory, a type or package named by the receiver,
2687
+ then a name declared once in the whole repo, which is a guess."""
2688
+ name = call.name
2689
+ cands = self.by_name.get((lang, name), [])
2690
+ types = self.types_by_name.get((lang, name), [])
2691
+ if len(self.repos) > 1: # another repository's names are seen only through what this file imports
2692
+ reach = self._reach(fid)
2693
+ mine = lambda i: _repo_of(i) == _repo_of(fid) or self.file_of.get(i) in reach
2694
+ cands, types = [c for c in cands if mine(c.id)], [t for t in types if mine(t)]
2695
+ if not cands and not types:
2696
+ return None
2697
+ ix = self._name_index(lang, name)
2698
+ r = call.receiver
2699
+
2700
+ def by_args(fns):
2701
+ """Overloads: keep those declared with as many parameters as the call passes, when that narrows it."""
2702
+ fns = list(fns)
2703
+ if len(fns) > 1:
2704
+ fns = [f for f in fns if not f.attrs.get("via_base")] or fns
2705
+ if len(fns) > 1 and call.argc >= 0:
2706
+ # A declaration with fewer parameters than the call passes cannot take it; of the rest, the one
2707
+ # with the fewest is likeliest (the others' extra parameters would need defaults).
2708
+ fit = [f for f in fns if f.attrs.get("params_seen") is None or f.attrs["params_seen"] >= call.argc]
2709
+ if fit and all(f.attrs.get("params_seen") is not None for f in fit):
2710
+ least = min(f.attrs["params_seen"] for f in fit)
2711
+ fit = [f for f in fit if f.attrs["params_seen"] == least]
2712
+ if len(fit) > 1 and call.args:
2713
+ fit = self._by_arg_types(lang, call, fit)
2714
+ return fit or fns
2715
+ return fns
2716
+
2717
+ def ctor_of(tids):
2718
+ out = []
2719
+ for t in tids:
2720
+ out += self.members.get(t, {}).get(name, []) or [m for c in CTORS for m in self.members.get(t, {}).get(c, [])]
2721
+ self.edges.append(Edge("instantiates", call.src_id, t, "heuristic"))
2722
+ return by_args(out) or None
2723
+ near_files = self._near(fid)
2724
+ here = self.nodes[fid].path.rpartition("/")[0]
2725
+ if r in (None, "this", "base"):
2726
+ chain = self._chain(call.enclosing_type)
2727
+ caller = self.nodes.get(call.src_id)
2728
+ if r is None and caller is not None and caller.attrs.get("explicit_self"):
2729
+ chain = [] # Go, Rust, Python: a member is reached only through the receiver
2730
+ for t in chain[1:] if r == "base" else chain:
2731
+ found = self.members.get(t, {}).get(name)
2732
+ if found:
2733
+ return by_args(found)
2734
+ if r is not None:
2735
+ return [] if cands else None
2736
+ local = ix["free_file"].get(fid) # declared in this file, then in this directory
2737
+ if local:
2738
+ return [local[0][1]]
2739
+ if here in ix["free_dir"]:
2740
+ return [ix["free_dir"][here]]
2741
+ if types and r is None:
2742
+ local_t = [t for t in types if self.file_of.get(t) == fid] or \
2743
+ [t for t in types if self.file_of.get(t) in near_files] or \
2744
+ [t for t in types if self.nodes[self.file_of.get(t, fid)].path.rpartition("/")[0] == here] or types
2745
+ if len(local_t) == 1:
2746
+ return ctor_of(local_t)
2747
+ free = ix["free"]
2748
+ imported = self._in_files(ix["free_file"], self.import_targets.get(fid, ())) or \
2749
+ self._in_files(ix["free_file"], near_files)
2750
+ if imported:
2751
+ return by_args(imported) if len({c.parent_id for c in imported}) == 1 else []
2752
+ if len(free) == 1 and (lang, name) not in self.outside_names:
2753
+ self._guessed = True
2754
+ return list(free)
2755
+ return [] if free else None
2756
+ if r not in ("?", None, "this", "base"):
2757
+ tid = self._generic_var_type(lang, call.src_id, r, call.enclosing_type)
2758
+ if tid == "external":
2759
+ return None
2760
+ if tid:
2761
+ for t in self._chain(tid):
2762
+ found = self.members.get(t, {}).get(name)
2763
+ if found:
2764
+ return by_args(found)
2765
+ if r == "?" and (call.receiver_type or call.chain is not None):
2766
+ # made on `new Foo()` or on what another call returns
2767
+ tid = (self.types_by_name.get((lang, call.receiver_type)) or [None])[0] if call.receiver_type else None
2768
+ if call.chain is not None and not call.receiver_type:
2769
+ rt = self._generic_returns(lang, call.chain)
2770
+ if rt and rt[0] is None and rt[1]:
2771
+ return None # a type from outside the repo
2772
+ tid = rt[0] if rt else None
2773
+ if tid:
2774
+ for t in self._chain(tid):
2775
+ found = self.members.get(t, {}).get(name)
2776
+ if found:
2777
+ return by_args(found)
2778
+ if r == "?":
2779
+ pool, owners = ix["typed"], ix["typed_owners"]
2780
+ if not pool:
2781
+ return None
2782
+ near = self._in_files(ix["typed_file"], near_files, fid)
2783
+ if near and len({c.parent_id for c in near}) == 1:
2784
+ self._guessed = True
2785
+ return by_args(near)
2786
+ else:
2787
+ # The receiver names a type (static call) or a package / module directory (Go's pkg.Func, Rust's mod::f).
2788
+ owned = [c for t in self.types_by_name.get((lang, r), []) for c in self.members.get(t, {}).get(name, [])]
2789
+ if owned:
2790
+ return by_args(owned)
2791
+ # Go's store.New(): the package the file imports under that name, before any other directory called store
2792
+ pkg = [c for c in self._in_files(ix["free_file"], self.import_targets.get(fid, ()))
2793
+ if (self.nodes[self.file_of[c.id]].path or "").rpartition("/")[0].rsplit("/", 1)[-1] == r]
2794
+ if pkg and len({self.file_of[c.id].rpartition("/")[0] for c in pkg}) == 1:
2795
+ return by_args(pkg)
2796
+ if r in ix["free_dir_base"]:
2797
+ return [ix["free_dir_base"][r]]
2798
+ # module.f() reaches the module's own functions before any method of that name in it
2799
+ if r in ix["stem_free"]:
2800
+ return [ix["stem_free"][r]]
2801
+ if r in ix["stem"]:
2802
+ return [ix["stem"][r]]
2803
+ if types and r not in self.types_by_name.get((lang, r), ()) and self._generic_var_type(lang, call.src_id, r, call.enclosing_type) is None:
2804
+ # module.Type(...): a type reached through the module or package the receiver names
2805
+ via = [t for t in types if self.file_of.get(t) in near_files
2806
+ or (self.nodes[self.file_of[t]].path.rpartition("/")[0].rsplit("/", 1)[-1] == r if t in self.file_of else False)]
2807
+ if len(via) == 1:
2808
+ return ctor_of(via)
2809
+ if r[:1].isupper() and not self.types_by_name.get((lang, r)):
2810
+ return None # a type from outside: Foo.bar()
2811
+ pool, owners = cands, ix["owners"]
2812
+ if (lang, name) in self.outside_names or len(name) <= 2:
2813
+ self.stats[BY_LANGUAGE[lang].NAME]["calls_guess_declined"] += 1
2814
+ return None
2815
+ if len(owners) == 1:
2816
+ self._guessed = True
2817
+ self.stats[BY_LANGUAGE[lang].NAME]["calls_by_unique_name"] += 1
2818
+ return list(pool)
2819
+ return []
2820
+
2821
+ def _resolve_call(self, lang: str, fid: str, call: CallSite) -> Optional[list[Node]]:
2822
+ """None = defined outside the workspace; [] = defined here but not pinned down."""
2823
+ if getattr(BY_LANGUAGE.get(lang), "GENERIC", False):
2824
+ return self._generic_call(lang, fid, call)
2825
+ name, argc = call.name, call.argc
2826
+ self._last_src = call.src_id
2827
+ if lang == "typescript": # one hop is too strict where a store or a module object sits between caller and function
2828
+ reach = self._reach(fid)
2829
+ defined_here = any(self.file_of.get(c.id) in reach for c in self.by_name.get((lang, name), ())) or bool(
2830
+ self.types_by_name.get((lang, name)))
2831
+ else:
2832
+ defined_here = any(self._can_see(fid, c.id) for c in self.by_name.get((lang, name), ())) or (
2833
+ lang in FILE_MODULE and bool(self.types_by_name.get((lang, name))))
2834
+ if name == ".ctor":
2835
+ tid = self._type(lang, call.receiver_type or "", call.src_id)
2836
+ if tid is None:
2837
+ return None
2838
+ ctors = self.members.get(tid, {}).get(".ctor")
2839
+ return self._pick(ctors, argc) if ctors else None
2840
+ if call.receiver is None:
2841
+ # Local functions of the caller, innermost first.
2842
+ cur = call.src_id
2843
+ while cur in self.nodes and self.nodes[cur].kind in ("callable", "test", "type"):
2844
+ if self.nodes[cur].kind == "type":
2845
+ if lang != "python":
2846
+ break
2847
+ cur = self.nodes[cur].parent_id # a method does not see its class's names, but does see the function around the class
2848
+ continue
2849
+ local = self.members.get(cur, {}).get(name)
2850
+ if local:
2851
+ return self._pick(local, argc)
2852
+ if lang == "python" and self.nodes.get(f"{cur}.{name}") is not None and self.nodes[f"{cur}.{name}"].kind == "type":
2853
+ return self._py_symbol(f"{cur}.{name}", argc) or [] # a class defined in the function
2854
+ cur = self.nodes[cur].parent_id
2855
+ if lang in FILE_MODULE:
2856
+ return self._py_bare(fid, call, defined_here)
2857
+ found = self._methods(call.enclosing_type, name, argc)
2858
+ if found:
2859
+ return found
2860
+ for tn in self.cs_static[fid]:
2861
+ found = self._methods(self._type(lang, tn, call.src_id), name, argc)
2862
+ if found:
2863
+ return found
2864
+ return [] if defined_here else None
2865
+ if lang == "typescript" and call.receiver in self.outside_imports[fid]:
2866
+ return None # React.useState(), path.join(): a namespace from outside
2867
+ inner = call.chain
2868
+ if lang == "python" and inner is not None and inner.attr and inner.chain is None and inner.receiver_type is None \
2869
+ and self._py_bound(fid, inner.src_id, inner.receiver or ""):
2870
+ target, symbol = self._py_name(fid, inner.src_id, inner.receiver)
2871
+ sub = self.py_modules[_repo_of(target)].get(f"{self._modpath(target)}.{inner.name}") if target and symbol is None else None
2872
+ if sub:
2873
+ return self._py_symbol(self._py_export(sub, name) or "", argc) # flask.json.dumps(): a submodule's function
2874
+ imported = (call.receiver_type is None and self._py_bound(fid, call.src_id, call.receiver or "")) if lang == "python" \
2875
+ else call.receiver in self.py_names[fid] # a typed local of the same name hides the import
2876
+ if lang in FILE_MODULE and call.receiver not in ("this",) and imported:
2877
+ target, symbol = self._py_name(fid, call.src_id, call.receiver) if lang == "python" else self.py_names[fid][call.receiver]
2878
+ if target is None:
2879
+ return [] if defined_here else None
2880
+ if symbol is None:
2881
+ return self._py_symbol(self._py_export(target, name) or "", argc)
2882
+ tid = self._py_export(target, symbol)
2883
+ if tid is None: # a module passed on under a name: `export * as ns from './format'` in a barrel
2884
+ mod = self._export_module(target, symbol)
2885
+ if mod:
2886
+ return self._py_symbol(self._py_export(mod, name) or "", argc)
2887
+ if tid is None and lang == "python":
2888
+ tid = self._py_var(target, symbol) # `from .globals import g`: a variable of a known type
2889
+ found = self._methods(tid, name, argc)
2890
+ return found or ([] if defined_here else None)
2891
+ tid, known = self._receiver_type(lang, call)
2892
+ if tid and lang == "python" and tid in self.nodes and self.nodes[tid].kind == "callable":
2893
+ return [self.nodes[tid]] if name == "__call__" else None # a function value: only calling it means anything
2894
+ if tid:
2895
+ # `self.go()` in a nested class is not the enclosing class's `go`: a receiver reaches its own type and bases
2896
+ found = self._methods(tid, name, argc, enclosing=False)
2897
+ if found:
2898
+ return found
2899
+ if lang == "csharp":
2900
+ found = self._extensions(lang, fid, tid, name, argc)
2901
+ if found:
2902
+ return found
2903
+ if not (lang == "typescript" and any(name in self.field_names.get(t, ()) for t in self._chain(tid))) and not (
2904
+ lang == "python" and any(self.nodes[t].attrs.get("open_base") for t in self._chain(tid) if t in self.nodes)):
2905
+ return None # the type is ours but the method is inherited from outside
2906
+ # A property that holds a function (`save: (x) => void` in an interface), or a Python class whose base
2907
+ # is made at run time: what is there is not known from the type, so fall through to the unique-name guess.
2908
+ tid, known = None, False
2909
+ if known:
2910
+ return None # receiver has a type that is not in the workspace
2911
+ if lang == "python" and self.outside_imports[fid]:
2912
+ root = call # sqlite3.connect(), os.path.join(): a module from outside
2913
+ while root.receiver == "?" and root.receiver_type is None and root.chain is not None and root.chain.attr:
2914
+ root = root.chain
2915
+ if root.receiver_type is None and root.chain is None and (root.receiver or "") in self.outside_imports[fid] \
2916
+ and not self._py_bound(fid, call.src_id, root.receiver):
2917
+ return None
2918
+ if not defined_here:
2919
+ return None
2920
+ if lang == "python" and name.startswith("__") and name.endswith("__"):
2921
+ return [] # __enter__, __call__: run implicitly on all kinds of objects, never guessed by name
2922
+ if (lang, name) in self.outside_names:
2923
+ self.stats[BY_LANGUAGE[lang].NAME]["calls_guess_declined"] += 1
2924
+ # In TypeScript most receivers are untyped, and a name seen on built-in types (push, get, keys) is
2925
+ # far more often the built-in than the one function here that shares it.
2926
+ return None if lang == "typescript" else []
2927
+ self.stats[BY_LANGUAGE[lang].NAME]["calls_by_unique_name"] += 1
2928
+ self._guessed = True
2929
+ # Receiver type unknown: accept only a name that is defined exactly once.
2930
+ cands = self._pick([c for c in self.by_name[(lang, name)] if (
2931
+ self.file_of.get(c.id) in self._reach(fid) if lang == "typescript" else self._can_see(fid, c.id))], argc)
2932
+ if lang == "python":
2933
+ # `con.close()` is a method: not a module's function (the receiver is no module), nor one nested in another
2934
+ cands = [c for c in cands if c.parent_id in self.nodes and self.nodes[c.parent_id].kind == "type"]
2935
+ owners = {c.parent_id for c in cands}
2936
+ if len(owners) > 1:
2937
+ # Several declarations: if all but one are overrides or implementations of the same
2938
+ # root declaration, the call is to that root (virtual dispatch).
2939
+ roots = {o for o in owners if not any(b in owners for b in self._chain(o)[1:])}
2940
+ if len(roots) == 1 and all(next(iter(roots)) in self._chain(o) for o in owners):
2941
+ root = roots.pop()
2942
+ return [c for c in cands if c.parent_id == root]
2943
+ return []
2944
+ return cands
2945
+
2946
+ def _resolve_overrides(self) -> None:
2947
+ """Link each method to the base or interface method it implements, by name and arity."""
2948
+ self.implementers: dict[str, list[str]] = defaultdict(list)
2949
+ linked = set()
2950
+
2951
+ def link(impl_id: str, base_method: str) -> None:
2952
+ if (impl_id, base_method) not in linked:
2953
+ linked.add((impl_id, base_method))
2954
+ self.edges.append(Edge("overrides", impl_id, base_method, "heuristic"))
2955
+ self.implementers[base_method].append(impl_id)
2956
+ for tid, bases in list(self.bases.items()):
2957
+ if not bases:
2958
+ continue
2959
+ chain = [base for base in self._chain(tid, enclosing=False)[1:]
2960
+ if self.nodes[base].kind == "type" and base != self.nodes[tid].parent_id]
2961
+ python = self.nodes[tid].language == "python"
2962
+ for name, impls in self.members.get(tid, {}).items():
2963
+ if name in (".ctor", ".dtor"):
2964
+ continue
2965
+ for impl in impls:
2966
+ if impl.kind != "callable":
2967
+ continue
2968
+ for base in chain:
2969
+ # Python and TypeScript have no overloads at run time: a method replaces the base's of that name,
2970
+ # whatever it takes
2971
+ cands = [c for c in self.members.get(base, {}).get(name, [])
2972
+ if impl.language in ("python", "typescript") or c.attrs.get("argc_max") == impl.attrs.get("argc_max")]
2973
+ if cands:
2974
+ link(impl.id, cands[0].id)
2975
+ # With several bases (a mixin and a base class), a Python method replaces the one
2976
+ # each of them declares: a call on the base class's `self` runs it too.
2977
+ if not python:
2978
+ break
2979
+ if not python:
2980
+ continue
2981
+ # A name the class inherits from a mixin shadows the same name on a later base: for a call from that
2982
+ # base's own code, the mixin's method is the one that runs.
2983
+ declared: dict[str, list[Node]] = defaultdict(list)
2984
+ for base in chain:
2985
+ for name, cands in self.members.get(base, {}).items():
2986
+ if name not in (".ctor", ".dtor") and name not in self.members.get(tid, {}):
2987
+ declared[name].extend(c for c in cands if c.kind == "callable")
2988
+ for name, methods in declared.items():
2989
+ for later in methods[1:]:
2990
+ if later.parent_id != methods[0].parent_id:
2991
+ link(methods[0].id, later.id)
2992
+
2993
+ # -- channels ------------------------------------------------------------
2994
+ def _event_field(self, type_id: Optional[str], name: str) -> Optional[str]:
2995
+ for t in self._chain(type_id):
2996
+ fid = f"{t}.{name}"
2997
+ n = self.nodes.get(fid)
2998
+ if n is not None and n.kind == "field" and n.attrs.get("native_kind") == "event":
2999
+ return fid
3000
+ return None
3001
+
3002
+ def _resolve_fields(self) -> None:
3003
+ """Link each function to the fields it reads and assigns."""
3004
+ field_id: dict[str, dict[str, str]] = defaultdict(dict)
3005
+ by_name: dict[tuple, list[str]] = defaultdict(list)
3006
+ for n in self.nodes.values():
3007
+ if n.kind == "field" and n.attrs.get("native_kind") not in ("enum_member", "event"):
3008
+ field_id[n.parent_id][n.name] = n.id
3009
+ by_name[(n.language, n.name)].append(n.id)
3010
+ found: dict[tuple, list] = {}
3011
+ for fid, res in self.results.items():
3012
+ lang = self.file_lang[fid]
3013
+ st = self.stats[BY_LANGUAGE[lang].NAME]
3014
+ for u in self._open(fid, res).field_uses:
3015
+ if u.src_id not in self.nodes:
3016
+ continue
3017
+ target, guessed = None, False
3018
+ if u.receiver in (None, "this"):
3019
+ if u.receiver is None and lang in FILE_MODULE:
3020
+ continue # a bare name in Python is a local or a global, never an attribute
3021
+ owners = self._chain(u.enclosing_type)
3022
+ elif u.receiver == "base":
3023
+ owners = self._chain(u.enclosing_type)[1:]
3024
+ else:
3025
+ probe = CallSite(u.src_id, u.name, u.receiver, u.receiver_type, 0, u.line, u.enclosing_type, chain=u.chain)
3026
+ tid, known = self._receiver_type(lang, probe)
3027
+ owners = self._chain(tid) if tid else []
3028
+ if lang == "csharp" and not tid and not known and len(u.name) > 3:
3029
+ # Receiver type unknown: accept only a field name declared exactly once, and say it is a guess.
3030
+ cands = [c for c in by_name.get((lang, u.name), ()) if self._can_see(fid, c)]
3031
+ if len(cands) == 1 and (lang, u.name) not in self.outside_names:
3032
+ target, guessed = cands[0], True
3033
+ for t in owners:
3034
+ if u.name in field_id.get(t, ()):
3035
+ target = field_id[t][u.name]
3036
+ break
3037
+ if lang == "python" and u.access != "w" and owners:
3038
+ self._read_getters(u, owners, target)
3039
+ if target is None or target == u.src_id:
3040
+ continue
3041
+ st["field_uses"] += 1
3042
+ for kind in (("reads",) if u.access == "r" else ("writes",) if u.access in ("w", "i") else ("reads", "writes")):
3043
+ slot = found.setdefault((kind, u.src_id, target), [0, u.line, False, 0])
3044
+ slot[0] += 1
3045
+ slot[1] = min(slot[1], u.line)
3046
+ slot[2] = slot[2] or guessed
3047
+ slot[3] += u.access == "i"
3048
+ self._shut()
3049
+ for (kind, src, dst), (n, line, guessed, init) in found.items():
3050
+ attrs = {"n": n, "line": line}
3051
+ if init == n:
3052
+ attrs["init"] = True # only ever set while creating the object, never changed afterwards
3053
+ self.edges.append(Edge(kind, src, dst, "guess" if guessed else "heuristic", attrs))
3054
+
3055
+ def _is_getter(self, fn: Node) -> bool:
3056
+ """A Python method that runs when its attribute is read: @property, or a decorator that is a descriptor
3057
+ class (one defining __get__, such as werkzeug's cached_property) declared somewhere in the workspace."""
3058
+ if fn.kind != "callable" or not fn.attrs.get("decorators"):
3059
+ return False
3060
+ if fn.id not in self._getter_memo:
3061
+ hit = False
3062
+ for d in fn.attrs["decorators"]:
3063
+ d = d.lstrip("@").split("(", 1)[0].strip()
3064
+ if d in ("property", "functools.cached_property", "abc.abstractproperty"):
3065
+ hit = True
3066
+ elif re.fullmatch(r"[A-Za-z_][\w.]*", d) and not d.endswith((".setter", ".deleter")):
3067
+ tid = self._type("python", d.rsplit(".", 1)[-1], fn.id)
3068
+ hit = bool(tid and any("__get__" in self.members.get(t, {}) for t in self._chain(tid)))
3069
+ if hit:
3070
+ break
3071
+ self._getter_memo[fn.id] = hit
3072
+ return self._getter_memo[fn.id]
3073
+
3074
+ def _read_getters(self, u: FieldUse, owners: list[str], field: Optional[str]) -> None:
3075
+ """Reading an attribute that a getter computes runs the getter, so the read is a call. On `self`, a
3076
+ subclass may compute an attribute the class itself holds as a plain value or a getter of its own
3077
+ (Flask's Request.max_content_length over Werkzeug's): those getters are reached too, as dispatch."""
3078
+ if not hasattr(self, "_getter_memo"):
3079
+ self._getter_memo: dict[str, bool] = {}
3080
+ self._subtypes: dict[str, set] = defaultdict(set)
3081
+ for sub, bases in self.bases.items():
3082
+ for b in bases:
3083
+ self._subtypes[b].add(sub)
3084
+ hits = []
3085
+ for t in owners:
3086
+ if field is not None and self.nodes[field].parent_id == t:
3087
+ break # a plain value nearer than any getter
3088
+ found = [m for m in self.members.get(t, {}).get(u.name, []) if self._is_getter(m)]
3089
+ if found:
3090
+ hits.append((found[0], "property"))
3091
+ break
3092
+ if u.receiver == "this" and (hits or field is not None):
3093
+ seen, queue = set(), [owners[0]]
3094
+ while queue:
3095
+ for sub in self._subtypes.get(queue.pop(), ()):
3096
+ if sub not in seen:
3097
+ seen.add(sub)
3098
+ queue.append(sub)
3099
+ hits += [(m, "virtual") for m in self.members.get(sub, {}).get(u.name, [])[:1] if self._is_getter(m)]
3100
+ for fn, dispatch in hits:
3101
+ if fn.id != u.src_id:
3102
+ self.calls.append((u.src_id, fn.id, dispatch, "heuristic", u.line))
3103
+
3104
+ def _resolve_events(self) -> None:
3105
+ """Link the code that raises an event to the code that handles it."""
3106
+ raisers: dict[str, list] = defaultdict(list)
3107
+ handlers: dict[str, list] = defaultdict(list)
3108
+ st = self.channel_stats["event"]
3109
+ for fid, res in self.results.items():
3110
+ lang = self.file_lang[fid]
3111
+ for ev in res.events:
3112
+ if ev.kind == "raise":
3113
+ field = self._event_field(ev.enclosing_type, ev.event)
3114
+ if field:
3115
+ raisers[field].append((ev.src_id, ev.line))
3116
+ continue
3117
+ if ev.receiver is None:
3118
+ field = self._event_field(ev.enclosing_type, ev.event)
3119
+ else:
3120
+ probe = CallSite(ev.src_id, ev.event, ev.receiver, ev.receiver_type, 0, ev.line, ev.enclosing_type)
3121
+ tid, _ = self._receiver_type(lang, probe)
3122
+ field = self._event_field(tid, ev.event) if tid else None
3123
+ if not field:
3124
+ if ev.handler is None or ev.receiver is not None:
3125
+ st["subscriptions_to_outside_events"] += 1
3126
+ continue
3127
+ target, via = ev.src_id, "lambda"
3128
+ if ev.handler:
3129
+ found = self._methods(ev.enclosing_type, ev.handler, 1) or self._methods(ev.enclosing_type, ev.handler, 0)
3130
+ if found:
3131
+ target, via = found[0].id, "method"
3132
+ handlers[field].append((target, via, ev.line, ev.src_id))
3133
+ seen = set()
3134
+ for field, subs in handlers.items():
3135
+ st["events_with_handlers"] += 1
3136
+ for raiser, _line in raisers.get(field, []):
3137
+ for target, via, line, subscriber in subs:
3138
+ key = (raiser, target, field)
3139
+ if key in seen or raiser not in self.nodes or target not in self.nodes:
3140
+ continue
3141
+ seen.add(key)
3142
+ st["links"] += 1
3143
+ self.edges.append(Edge("communicates", raiser, target, "heuristic", {
3144
+ "channel": "event", "address": field, "direction": "push", "handler": via,
3145
+ "subscriber": subscriber, "subscribed_at": line}))
3146
+ if field not in raisers:
3147
+ st["events_never_raised_here"] += 1
3148
+ st["events_declared"] = sum(1 for n in self.nodes.values()
3149
+ if n.kind == "field" and n.attrs.get("native_kind") == "event")
3150
+
3151
+ GENERIC_DIRS = {"data", "temp", "file", "files", "json", "path", "output", "outputs", "input", "home", "user", "users",
3152
+ "test", "tests", "docs", "static", "assets", "resources", "config", "build", "dist", "local"}
3153
+
3154
+ def _route_handlers(self, routes: list) -> tuple[list, dict]:
3155
+ """The functions that answer routes. An inline handler is its own function (an adapter made it, nested in
3156
+ the function that registers the route): the registrar registers it, which the map keeps as a call so that
3157
+ whatever runs the registrar still reaches the handler. A handler given by name (server.get("/x", list))
3158
+ is the function that name refers to, when the registrar's reference to it was resolved.
3159
+ Returns the routes with their serving end moved to the handler, and each handler's registrar."""
3160
+ handlers: dict[str, str] = {}
3161
+ for n in list(self.nodes.values()):
3162
+ if n.kind == "callable" and n.attrs.get("native_kind") == "route_handler" and n.parent_id in self.nodes:
3163
+ handlers[n.id] = n.parent_id
3164
+ self.calls.append((n.parent_id, n.id, "registers", "heuristic", n.attrs.get("registered_at") or n.span_start))
3165
+ named = [i for i, (e, _) in enumerate(routes) if e.handler]
3166
+ if named:
3167
+ refs: dict[str, set] = defaultdict(set)
3168
+ wanted = {routes[i][0].src_id for i in named}
3169
+ for src, dst, dispatch, _p, _l in self.calls:
3170
+ if dispatch == "reference" and src in wanted:
3171
+ refs[src].add(dst)
3172
+ routes = list(routes)
3173
+ for i in named:
3174
+ e, res = routes[i]
3175
+ name = e.handler.rsplit(".", 1)[-1]
3176
+ hits = {d for d in refs.get(e.src_id, ()) if d in self.nodes and self.nodes[d].name == name}
3177
+ if len(hits) == 1:
3178
+ target = hits.pop()
3179
+ handlers.setdefault(target, e.src_id)
3180
+ routes[i] = (Endpoint(e.channel, e.role, target, e.address, e.line, e.method, e.literals, e.handler), res)
3181
+ return routes, handlers
3182
+
3183
+ @staticmethod
3184
+ def _one_handler(cands: list, handlers: dict, method: Optional[str], literal) -> list:
3185
+ """A request that several handlers' routes fit. A route that names more of the path outright wins
3186
+ (/api/review/health over /api/review/:id), as routers pick it; then, for a request whose method is not
3187
+ written, a GET, which is what a request sent with no method is; failing both, the function that registers
3188
+ them all answers it, as it did before handlers had functions of their own. Empty when none settles it."""
3189
+ most = max(literal(r) for r in cands)
3190
+ cands = [r for r in cands if literal(r) == most]
3191
+ if len({r.src_id for r in cands}) == 1:
3192
+ return cands
3193
+ gets = [r for r in cands if r.method in (None, "GET")] if method is None else []
3194
+ if gets and len({r.src_id for r in gets}) == 1:
3195
+ return gets
3196
+ owners = {handlers.get(r.src_id) for r in cands}
3197
+ if len(owners) == 1 and None not in owners:
3198
+ r, same = cands[0], {c.method for c in cands}
3199
+ return [Endpoint(r.channel, r.role, owners.pop(), r.address, r.line, same.pop() if len(same) == 1 else None,
3200
+ r.literals)]
3201
+ return []
3202
+
3203
+ def _resolve_endpoints(self) -> None:
3204
+ """Link the two ends of channels that are not calls: an HTTP request to its route, and code that
3205
+ writes a file to code that reads it."""
3206
+ http, files = self.channel_stats["http"], self.channel_stats["file"]
3207
+
3208
+ def root_of(i): # the outermost function a nested function or inline test sits in
3209
+ cur = i
3210
+ while self.nodes[cur].parent_id in self.nodes and self.nodes[self.nodes[cur].parent_id].kind in ("callable", "test"):
3211
+ cur = self.nodes[cur].parent_id
3212
+ return cur
3213
+
3214
+ def segments(path):
3215
+ path = re.sub(r"^[a-z]+://[^/]+", "", path).split("?", 1)[0].split("#", 1)[0]
3216
+ return [x for x in path.split("/") if x]
3217
+ routes, requests, io = [], [], []
3218
+ for res in self.results.values():
3219
+ for e in res.endpoints:
3220
+ if e.src_id not in self.nodes or e.channel not in ("http", "file"):
3221
+ continue # the other channels are linked in leyline.channels
3222
+ (routes if (e.channel, e.role) == ("http", "serve") else requests if e.channel == "http" else io).append((e, res))
3223
+ http["routes"], http["requests"] = len(routes), sum(1 for e, _ in requests if e.role != "maybe")
3224
+ routes, handlers = self._route_handlers(routes)
3225
+
3226
+ def fits(route, request):
3227
+ a, b = segments(route.address), segments(request.address)
3228
+ if route.method and request.method and route.method != request.method:
3229
+ return False
3230
+ if a and _wildcard(a[-1]):
3231
+ return _fits_wildcard(a, b)
3232
+ if len(a) != len(b):
3233
+ return False
3234
+ # ASP.NET's [controller] is the class's name, which clients write in lower case
3235
+ return all(_segment_fits(x, y) for x, y in zip(a, b))
3236
+ seen = set()
3237
+ for req, _ in requests:
3238
+ cands = [r for r, _ in routes if fits(r, req)]
3239
+ # In a workspace, a route in the request's own repository wins over one in another.
3240
+ cands = [r for r in cands if _repo_of(r.src_id) == _repo_of(req.src_id)] or cands
3241
+ near = [r for r in cands if root_of(r.src_id) == root_of(req.src_id)] # a route declared inside the same test
3242
+ chosen = near or (cands if len({r.src_id for r in cands}) == 1 else [])
3243
+ literal = lambda r: sum(1 for x in segments(r.address) if not x.startswith(("<", "{", ":")) and not _wildcard(x))
3244
+ if handlers and len({r.src_id for r in chosen or cands}) > 1: # several handlers, perhaps of one registrar
3245
+ chosen = self._one_handler(chosen or cands, handlers, req.method, literal) or chosen
3246
+ elif len({r.src_id for r in chosen or cands}) > 1:
3247
+ # In every language, the route that names more of the path outright wins: /items/special over /items/{id}.
3248
+ most = max(literal(r) for r in chosen or cands)
3249
+ top = [r for r in chosen or cands if literal(r) == most]
3250
+ chosen = top if len({r.src_id for r in top}) == 1 else chosen
3251
+ if req.role == "maybe": # a path handed to a wrapper: a request only when exactly one route serves it
3252
+ chosen = chosen if len(segments(req.address)) >= 2 and len({r.src_id for r in chosen}) == 1 else []
3253
+ elif cands and not chosen:
3254
+ http["ambiguous"] += 1
3255
+ for r in chosen:
3256
+ # one link per route called: a function that calls several routes of one handler (a minimal API's
3257
+ # registrar, a router) is linked for each of them
3258
+ key = (req.src_id, r.src_id, r.address, req.method or r.method)
3259
+ if key not in seen and req.src_id != r.src_id:
3260
+ seen.add(key)
3261
+ http["links"] += 1
3262
+ self.edges.append(Edge("communicates", req.src_id, r.src_id, "heuristic",
3263
+ {"channel": "http", "address": f"{req.method or r.method or 'ANY'} {r.address}", "line": req.line,
3264
+ **({"handler": True} if r.src_id in handlers else {})}))
3265
+
3266
+ # Files: a writer and a reader are linked when the path fragments written in each agree.
3267
+ def tokens(strings):
3268
+ dirs, exts, names = set(), set(), set()
3269
+ for s in strings:
3270
+ s = s.replace("\\", "/")
3271
+ if len(s) > 160 or any(ch.isspace() for ch in s):
3272
+ continue
3273
+ parts = [x for x in s.split("/") if x not in ("", ".", "..")]
3274
+ for i, seg in enumerate(parts):
3275
+ clean = re.sub(r"\{[^}]*\}|\*", "", seg)
3276
+ if i == len(parts) - 1 and "." in seg:
3277
+ stem, _, ext = clean.partition(".")
3278
+ if re.fullmatch(r"[A-Za-z0-9_]{1,10}(\.[A-Za-z0-9_]{1,10})?", ext or ""):
3279
+ exts.add("." + ext.lower())
3280
+ if stem and clean == seg and len(stem) > 2:
3281
+ names.add(seg.lower())
3282
+ elif re.fullmatch(r"[A-Za-z][A-Za-z0-9_\-]{3,}", seg) and seg.lower() not in self.GENERIC_DIRS:
3283
+ dirs.add(seg.lower())
3284
+ return dirs, exts, names
3285
+ by_fn: dict[tuple, list] = defaultdict(list)
3286
+ for e, res in io:
3287
+ by_fn[(e.src_id, e.role)].extend(e.literals + res.path_strings.get(e.src_id, []) + res.path_strings.get(root_of(e.src_id), []))
3288
+ files["read_sites"] = sum(1 for e, _ in io if e.role == "read")
3289
+ files["write_sites"] = sum(1 for e, _ in io if e.role == "write")
3290
+ toks = {k: tokens(v) for k, v in by_fn.items()}
3291
+ pairs = []
3292
+ for (w, wr), (wd, we, wn) in toks.items():
3293
+ if wr != "write":
3294
+ continue
3295
+ for (r, rr), (rd, re_, rn) in toks.items():
3296
+ if rr != "read" or root_of(r) == root_of(w) or _repo_of(r) != _repo_of(w):
3297
+ continue # file paths are matched within one repository; across two they say too little
3298
+ shared_name = wn & rn
3299
+ shared = (wd & rd, we & re_)
3300
+ if shared_name:
3301
+ pairs.append((w, r, sorted(shared_name)[0]))
3302
+ elif shared[0] and shared[1]:
3303
+ pairs.append((w, r, f"{sorted(shared[0], key=lambda x: (-len(x), x))[0]}/*{sorted(shared[1], key=lambda x: (-len(x), x))[0]}"))
3304
+ fan = Counter(p[2] for p in pairs)
3305
+ for w, r, address in pairs:
3306
+ if fan[address] > 8:
3307
+ files["too_common_to_link"] += 1
3308
+ continue # a fragment this common says nothing about who reads whose file
3309
+ files["links"] += 1
3310
+ self.edges.append(Edge("communicates", w, r, "guess", {"channel": "file", "address": address}))
3311
+
3312
+ def _resolve_spawns(self) -> None:
3313
+ """Link code that launches a program to that program's entry point, when it is in the workspace."""
3314
+ st = self.channel_stats["process"]
3315
+ entries: dict[str, list[str]] = defaultdict(list) # module -> cli entry callables
3316
+ for e in self.edges:
3317
+ if e.kind == "exposes" and self.nodes[e.src_id].attrs.get("trigger") == "cli":
3318
+ mod = self.nodes[self.file_of[e.dst_id]].parent_id if e.dst_id in self.file_of else None
3319
+ if mod:
3320
+ entries[mod].append(e.dst_id)
3321
+ files_by_path = {(_repo_of(n.id), n.path): n.id for n in self.nodes.values() if n.kind == "file"}
3322
+ modules = [(n.path, n.id) for n in self.nodes.values() if n.kind == "module" and n.path]
3323
+
3324
+ def match(text: str, repo: str):
3325
+ text = text.strip().replace("\\", "/")
3326
+ if (repo, text) in files_by_path: # a script path, in the launching file's repository
3327
+ fid = files_by_path[(repo, text)]
3328
+ tops = [e.dst_id for e in self.edges if e.kind == "exposes" and self.file_of.get(e.dst_id) == fid]
3329
+ return tops[0] if tops else fid
3330
+ built = re.match(r"^(.*?/)?(?:dist|build|out|lib)/(.+)\.[cm]?js$", text)
3331
+ if built: # a built script (host/dist/index.js) runs the source it was built from (host/src/index.ts)
3332
+ for src in ("src/", ""):
3333
+ for ext in (".ts", ".mts", ".cts", ".tsx", ".js", ".mjs"):
3334
+ source = f"{built.group(1) or ''}{src}{built.group(2)}{ext}"
3335
+ if (repo, source) in files_by_path:
3336
+ return match(source, repo)
3337
+ for path, mid in sorted(modules, key=lambda m: _repo_of(m[1]) != repo): # own repository first
3338
+ stem = text.rsplit("/", 1)[-1].rsplit(".", 1)[0] if "." in text.rsplit("/", 1)[-1] else None
3339
+ if text == path or text.startswith(path + "/") or (stem and stem == path.rsplit("/", 1)[-1] and text.endswith((".dll", ".exe", ".csproj"))):
3340
+ if not entries.get(mid):
3341
+ continue
3342
+ return entries[mid][0] if len(entries[mid]) == 1 else mid
3343
+ return None
3344
+
3345
+ seen = set()
3346
+ for fid, res in self.results.items():
3347
+ for sp in res.spawns:
3348
+ st["launch_sites"] += 1
3349
+ hits = [(match(s, _repo_of(fid)), s, "heuristic") for s in sp.strings]
3350
+ hits = [h for h in hits if h[0]]
3351
+ if not hits:
3352
+ hits = [(match(s, _repo_of(fid)), s, "guess") for s in dict.fromkeys(sp.file_strings)]
3353
+ hits = [h for h in hits if h[0]]
3354
+ if not hits:
3355
+ st["launches_of_outside_programs"] += 1
3356
+ continue
3357
+ for target, text, precision in dict.fromkeys(hits):
3358
+ if (sp.src_id, target) in seen or target == sp.src_id:
3359
+ continue
3360
+ seen.add((sp.src_id, target))
3361
+ st["links"] += 1
3362
+ self.edges.append(Edge("communicates", sp.src_id, target, precision, {
3363
+ "channel": "process", "address": text,
3364
+ "direction": "both" if sp.pipes else "start", "pipes": sp.pipes, "launched_at": sp.line}))
3365
+
3366
+ # -- flows ---------------------------------------------------------------
3367
+ def _dispatch_reaches(self, home: str, impl: str) -> bool:
3368
+ """Can a program whose entry is in `home` be running this implementation of an interface?"""
3369
+ if self._visible(home) is not None:
3370
+ return self._can_see(home, impl)
3371
+ # A loose file with no project: it can only be composed of the modules its own module imports.
3372
+ mod = self.nodes[home].parent_id
3373
+ if mod not in self._loose_reach:
3374
+ files = {f for f in self.file_lang if self.nodes[f].parent_id == mod}
3375
+ self._loose_reach[mod] = {mod} | {e.dst_id for e in self.edges if e.kind == "imports" and e.src_id in files}
3376
+ target = self.file_of.get(impl)
3377
+ return target is not None and self.nodes[target].parent_id in self._loose_reach[mod]
3378
+
3379
+ def _build_flows(self, max_depth: int = 8, max_steps: int = 300) -> None:
3380
+ """Walk the call graph from every entry point and test, in source order."""
3381
+ out: dict[str, list] = defaultdict(list)
3382
+ for src, dst, _disp, _prec, line in self.calls:
3383
+ out[src].append(((line or 0) + self.call_col.get((src, dst, line), 0) / 10000, dst, "calls", None))
3384
+ if not self.keep_results:
3385
+ self.call_col.clear() # read only here (an entry per call): the walk's steps reuse its memory
3386
+ for e in self.edges:
3387
+ if e.kind == "communicates":
3388
+ a = e.attrs or {}
3389
+ if a.get("channel") in ("file", "db", "format"):
3390
+ continue # writing a file, a row or a key does not run whoever reads it later
3391
+ # A registered implementation is listed ahead of the other implementations of its interface.
3392
+ order = 10 ** 9 - 2 if a.get("channel") == "di" and "registered_in" in a else 10 ** 9
3393
+ out[e.src_id].append((a.get("launched_at") or a.get("line") or order, e.dst_id, a.get("channel", "channel"),
3394
+ a.get("subscriber")))
3395
+ # A call to an interface or base method may land in any implementation.
3396
+ for base, impls in self.implementers.items():
3397
+ for impl in impls:
3398
+ out[base].append((10 ** 9 - 1, impl, "dispatch", None))
3399
+ # A test declared inline runs where its runner call sits.
3400
+ for n in self.nodes.values():
3401
+ if n.kind == "test" and n.parent_id:
3402
+ out[n.parent_id].append((n.span_start or 0, n.id, "runs", None))
3403
+ for lst in out.values():
3404
+ lst.sort(key=lambda x: (x[0], x[1]))
3405
+ starts = []
3406
+ for e in self.edges:
3407
+ if e.kind == "exposes":
3408
+ ep = self.nodes[e.src_id]
3409
+ starts.append((e.dst_id, "entry", ep.attrs.get("trigger")))
3410
+ for n in self.nodes.values():
3411
+ if n.kind == "test" or n.attrs.get("is_test"):
3412
+ starts.append((n.id, "test", n.attrs.get("framework")))
3413
+ starts = flow_ids(starts)
3414
+ keep = self._flow_select(out, starts)
3415
+ for start, kind, detail, fid in starts:
3416
+ if start not in self.nodes or (keep is not None and start not in keep):
3417
+ continue
3418
+ entry_file = self.file_of.get(start)
3419
+ seen, steps, truncated = {start}, [(0, 0, start, "start", None, None)], False
3420
+ stack = [(start, 0, 0)]
3421
+ # Depth-first, pre-order, each callable listed once per flow.
3422
+ def walk(node, depth, parent_seq, home):
3423
+ nonlocal truncated
3424
+ if depth >= max_depth:
3425
+ if out.get(node):
3426
+ truncated = True
3427
+ return
3428
+ for line, dst, via, subscriber in out.get(node, ()):
3429
+ if dst in seen or dst not in self.nodes:
3430
+ continue
3431
+ if via == "event" and subscriber not in seen:
3432
+ continue # nobody in this flow subscribed, so the handler does not run here
3433
+ if via in ("dispatch", "di") and home and self.file_lang.get(home) == "csharp" \
3434
+ and not self._dispatch_reaches(home, dst):
3435
+ continue # an implementation in a project this program does not reference
3436
+ if kind == "test" and via == "runs":
3437
+ continue
3438
+ if len(steps) >= max_steps:
3439
+ truncated = True
3440
+ return
3441
+ seen.add(dst)
3442
+ seq = len(steps)
3443
+ steps.append((seq, depth + 1, dst, via, int(line) if line < 10 ** 9 - 1 else None, parent_seq))
3444
+ # A launched program is its own composition: what it can reach is judged from there.
3445
+ walk(dst, depth + 1, seq, self.file_of.get(dst, home) if via == "process" else home)
3446
+ walk(start, 0, 0, entry_file)
3447
+ n = self.nodes[start]
3448
+ name = _test_title(n) if n.kind == "test" else start.split(":", 2)[-1].split("::")[-1]
3449
+ mods = {self.nodes[self.file_of[s[2]]].parent_id for s in steps if s[2] in self.file_of}
3450
+ self.flows.append((fid, name, "static", start, 0.0, None, "fact", SOURCE, {
3451
+ "kind": kind, "detail": detail, "steps": len(steps), "truncated": truncated,
3452
+ "modules": sorted(m for m in mods if m)}))
3453
+ self.flow_steps.add(fid, steps)
3454
+
3455
+ def _flow_select(self, out: dict, starts: list) -> Optional[set]:
3456
+ """The starts to walk, given the call graph the walk reads (`out`); None for all of them.
3457
+ leyline.incremental walks again only the flows an edit can have changed."""
3458
+ return None
3459
+
3460
+ def _py_symbol(self, node_id: str, argc: int) -> Optional[list[Node]]:
3461
+ n = self.nodes.get(node_id)
3462
+ if n is None:
3463
+ return None
3464
+ if n.kind == "callable":
3465
+ return [n] if argc < 0 else (self._pick([n], argc) or [n])
3466
+ if n.kind == "type":
3467
+ init = next((found for c in CTORS for found in [self._methods(n.id, c, argc)] if found), None)
3468
+ self.edges.append(Edge("instantiates", self._last_src, n.id, "heuristic"))
3469
+ return init or None
3470
+ return None
3471
+
3472
+ def _py_bare(self, fid: str, call: CallSite, defined_here: bool) -> Optional[list[Node]]:
3473
+ self._last_src = call.src_id
3474
+ lang = self.file_lang[fid]
3475
+ own = f"{_repo_of(fid)}:{lang}:{self._modpath(fid)}.{call.name}"
3476
+ hit = self._py_symbol(own, call.argc)
3477
+ if hit is not None:
3478
+ return hit
3479
+ if self._py_bound(fid, call.src_id, call.name):
3480
+ target, symbol = self._py_name(fid, call.src_id, call.name)
3481
+ return self._py_symbol(self._py_export(target, symbol or call.name) or "", call.argc) if target else None
3482
+ if lang == "typescript" and own not in self.nodes and call.name not in self.outside_imports[fid]:
3483
+ # Not declared here and not imported by name: a local taken out of something (`const { save } = await
3484
+ # load()`). If exactly one exported function reachable from this file has the name, say so as a guess.
3485
+ reach = self._reach(fid)
3486
+ cands = [c for c in self.by_name.get((lang, call.name), ())
3487
+ if self.file_of.get(c.id) in reach and c.parent_id == self.file_of.get(c.id) and c.attrs.get("visibility") == "public"]
3488
+ if len(cands) == 1 and len(call.name) > 3:
3489
+ self._guessed = True
3490
+ self.stats[BY_LANGUAGE[lang].NAME]["calls_by_unique_name"] += 1
3491
+ return cands
3492
+ return None # builtins, star imports, and classes with no constructor
3493
+
3494
+ # -- write ---------------------------------------------------------------
3495
+ def _owner(self, node_id: str) -> str:
3496
+ """The repository a node's rows are written under."""
3497
+ r = _repo_of(node_id)
3498
+ return r if r in self.repos else self.repo
3499
+
3500
+ def _final(self) -> tuple[list, list]:
3501
+ """The edges and calls as they are written: edges whose ends both exist, each once, links between
3502
+ repositories marked, and a contains edge per parent; calls whose caller exists, each once, sorted.
3503
+ Made once per run."""
3504
+ got = getattr(self, "_final_rows", None)
3505
+ if got is not None:
3506
+ return got
3507
+ multi, owner = len(self.repos) > 1, self._owner
3508
+ # Drop edges whose endpoints did not survive (defensive) and exact duplicates.
3509
+ seen, edges = set(), []
3510
+ for e in self.edges:
3511
+ key = (e.kind, e.src_id, e.dst_id, tuple(sorted((e.attrs or {}).items(), key=str)).__repr__())
3512
+ if e.src_id in self.nodes and e.dst_id in self.nodes and key not in seen:
3513
+ seen.add(key)
3514
+ if multi and owner(e.dst_id) != owner(e.src_id):
3515
+ e.attrs = {**(e.attrs or {}), "to_repo": owner(e.dst_id)} # a link between repositories
3516
+ edges.append(e)
3517
+ for n in self.nodes.values():
3518
+ if n.parent_id:
3519
+ edges.append(Edge("contains", n.parent_id, n.id, "exact", None)) # no attrs: not an empty dict per node
3520
+ calls = sorted({c for c in self.calls if c[0] in self.nodes})
3521
+ self._final_rows = (edges, calls)
3522
+ return self._final_rows
3523
+
3524
+ def _write(self, con) -> None:
3525
+ owner = self._owner
3526
+ with con:
3527
+ for repo, root in self.repos.items():
3528
+ store.clear_facts(con, repo)
3529
+ store.write_nodes(con, [n for n in self.nodes.values() if owner(n.id) == repo], repo, SOURCE, self.commits[repo])
3530
+ store.set_root(con, repo, root)
3531
+ edges, calls = self._final()
3532
+ for repo in self.repos: # each row carries the commit of the repository its source is in
3533
+ store.write_edges(con, [e for e in edges if owner(e.src_id) == repo], SOURCE, self.commits[repo])
3534
+ store.write_calls(con, [c for c in calls if owner(c[0]) == repo], self.commits[repo])
3535
+ store.write_flows(con, self.repo, self.flows, self.flow_steps)
3536
+ self._write_coverage(con, len(self.flows), len(self.flow_steps))
3537
+ store.rebuild_derived(con)
3538
+
3539
+ def _write_coverage(self, con, flows: int, steps: int) -> None:
3540
+ """What each extractor did, per repo, with the run's counts."""
3541
+ multi = len(self.repos) > 1
3542
+ from .adapters import ADAPTERS
3543
+ # Counts are for the whole run; in a workspace every member repo gets the same rows, marked as such.
3544
+ ws = {"workspace": sorted(self.repos)} if multi else {}
3545
+ for repo in self.repos:
3546
+ commit = self.commits[repo]
3547
+ for a in ADAPTERS:
3548
+ st = dict(self.stats.get(a.NAME, {}))
3549
+ status = "ok" if st.get("files") else "no_files"
3550
+ store.write_coverage(con, repo, a.NAME, a.VERSION, status, commit, {**st, **ws} if st else st)
3551
+ for channel in ("event", "process", "di", "http", "rpc", "queue", "db", "file", "format"):
3552
+ store.write_coverage(con, repo, f"communicates:{channel}", "0.1", "ok", commit,
3553
+ dict(self.channel_stats.get(channel, {})))
3554
+ store.write_coverage(con, repo, "flows:static", "0.1", "ok", commit, {"flows": flows, "steps": steps, **ws})
3555
+ for name in ("exact:roslyn", "exact:scip"):
3556
+ info = dict(self.exact_stats.get(name, {}))
3557
+ status = info.pop("status", "not_analyzed")
3558
+ store.write_coverage(con, repo, name, "1" if status == "ok" else "-",
3559
+ status if status in ("ok", "failed") else "not_analyzed", commit, info)
3560
+ kept = con.execute("SELECT format, stats FROM coverage_runs ORDER BY created DESC LIMIT 1").fetchone()
3561
+ if kept: # an imported coverage file outlives a re-index
3562
+ store.write_coverage(con, repo, "coverage", kept["format"], "ok", commit, json.loads(kept["stats"] or "{}"))
3563
+ else:
3564
+ store.write_coverage(con, repo, "coverage", "-", "not_analyzed", commit, {})
3565
+ if multi:
3566
+ con.execute("INSERT OR REPLACE INTO meta VALUES ('workspace', ?)", (json.dumps(list(self.repos)),))
3567
+
3568
+
3569
+ def _normalize(parts: tuple) -> list[str]:
3570
+ out: list[str] = []
3571
+ for p in parts:
3572
+ if p == "..":
3573
+ if out:
3574
+ out.pop()
3575
+ elif p not in (".", ""):
3576
+ out.append(p)
3577
+ return out
3578
+
3579
+
3580
+ def first_commit(root: Path) -> Optional[str]:
3581
+ """The repository's first commit (the earliest of its root commits), which a moved or renamed clone keeps."""
3582
+ out = _git(root, "rev-list", "--max-parents=0", "HEAD", timeout=60)
3583
+ return sorted(out.split())[0] if out else None
3584
+
3585
+
3586
+ def _moved_members(con, given: list[tuple[Path, str]], stored: list[str]) -> list[tuple[Path, str]]:
3587
+ """A directory named to map that is a stored member moved or renamed keeps that member's id, so its code is
3588
+ not added a second time. It is that member when the member's directory is gone and both are git repositories
3589
+ with the same first commit and, where both have an origin, the same origin. Anything less sure (no git, a
3590
+ store that never recorded the first commit) is a new member; `leyline map --forget <id>` drops the old one."""
3591
+ held = store.roots(con)
3592
+ names = {rid for _, rid in given}
3593
+ gone = [rid for rid in stored if rid not in names and not (held.get(rid) and Path(held[rid]).is_dir())]
3594
+ if not gone:
3595
+ return given
3596
+ out = []
3597
+ for root, rid in given:
3598
+ if rid not in stored and root not in {Path(p).resolve() for p in held.values()}:
3599
+ first = first_commit(root)
3600
+ url = _git(root, "remote", "get-url", "origin")
3601
+ for old in list(gone):
3602
+ was = con.execute("SELECT value FROM meta WHERE key = ?", (f"first_commit:{old}",)).fetchone()
3603
+ attrs = con.execute("SELECT attrs FROM nodes WHERE id = ? AND kind = 'repo'", (old,)).fetchone()
3604
+ old_url = (json.loads(attrs[0] or "{}") if attrs else {}).get("url")
3605
+ if first and was and was[0] == first and (not url or not old_url or url == old_url):
3606
+ print(f"leyline: {root.name} is workspace member {old}, moved from {held.get(old)}; mapped as {old}",
3607
+ file=sys.stderr)
3608
+ rid = old
3609
+ gone.remove(old)
3610
+ break
3611
+ out.append((root, rid))
3612
+ return out
3613
+
3614
+
3615
+ def workspace(con, roots: list[str | Path], repo_id: Optional[str] = None) -> list[tuple[Path, str]]:
3616
+ """The (root, repo id) pairs to index together. Repositories named together make the store a workspace,
3617
+ and it remembers them: indexing any one of them later indexes all of them again, so the links between
3618
+ them are kept. A member whose directory is gone is left as it was stored."""
3619
+ given = [(Path(r).resolve(), repo_id if repo_id and len(roots) == 1 else Path(r).resolve().name) for r in roots]
3620
+ row = con.execute("SELECT value FROM meta WHERE key = 'workspace'").fetchone()
3621
+ stored = json.loads(row[0]) if row else []
3622
+ if len(given) == 1 and not stored:
3623
+ return given
3624
+ given = _moved_members(con, given, stored)
3625
+ out = list(given)
3626
+ for rid in stored:
3627
+ here = store.roots(con).get(rid)
3628
+ root = (str(here),) if here is not None else None
3629
+ if any(rid == g[1] or (root and Path(root[0]) == g[0]) for g in given):
3630
+ continue
3631
+ if root and Path(root[0]).is_dir():
3632
+ out.append((Path(root[0]), rid))
3633
+ else:
3634
+ print(f"leyline: workspace member {rid} is not at {root[0] if root else '?'}; its facts are kept as they were",
3635
+ file=sys.stderr)
3636
+ ids = [rid for _, rid in out]
3637
+ dupes = sorted({i for i in ids if ids.count(i) > 1})
3638
+ if dupes:
3639
+ raise ValueError(f"two repositories would have the id {dupes[0]!r}; index them from directories with different names")
3640
+ return out
3641
+
3642
+
3643
+ def index(root: str | Path | list, db_path: str | Path, repo_id: Optional[str] = None, exact: str = "off",
3644
+ scip: Optional[list[str]] = None, full: bool = False, _verify: bool = True,
3645
+ listed: Optional[dict] = None) -> dict:
3646
+ """Index a repository, or several as one workspace (`root` a list). `exact` is off, auto, roslyn or scip:
3647
+ whether a compiler's view of the references replaces the syntax-based one (see leyline.exact). `scip`
3648
+ lists index.scip files. When the store was made by an earlier run, only what changed since is done again
3649
+ (leyline.incremental), and the store comes out as a full run would leave it; `full` makes it a full run."""
3650
+ from . import incremental, tours
3651
+
3652
+ con = store.connect(db_path)
3653
+ inc = None
3654
+ try:
3655
+ members = workspace(con, list(root) if isinstance(root, (list, tuple)) else [root], repo_id)
3656
+ ix = Indexer(members[0][0], members[0][1], members[1:])
3657
+ ix.listed = dict(listed or {}) # repo id -> its files or its Listing, when the caller has listed them
3658
+ ix.exact_mode, ix.scip_paths = ("scip" if scip and exact == "off" else exact), list(scip or [])
3659
+ ix.keep_results = False
3660
+ inc = incremental.Run(con, db_path, ix, full=full)
3661
+ began = time.perf_counter()
3662
+ # The run makes millions of objects that live until it ends; the cycle collector would scan them over and
3663
+ # over (a third of loading a large repository's cached parse output) and find nothing to free.
3664
+ gc.disable()
3665
+ try:
3666
+ stats = ix.run(con)
3667
+ finally:
3668
+ gc.enable()
3669
+ stats.update(ix.exact_stats)
3670
+ left = {r: skipped_summary(ix.skipped.get(r, []), ix.failed.get(r, [])) for r in ix.repos}
3671
+ stats["left_out"] = left if len(ix.repos) > 1 else left[ix.repo]
3672
+ with con: # so the overview and the map page can say what is not in the map
3673
+ for r, v in left.items():
3674
+ con.execute("INSERT OR REPLACE INTO meta VALUES (?, ?)", (f"left_out:{r}", json.dumps(v)))
3675
+ for r, r_root in ix.repos.items(): # who each member is, to know it again if its folder moves
3676
+ if con.execute("SELECT 1 FROM meta WHERE key = ?", (f"first_commit:{r}",)).fetchone() is None:
3677
+ first = first_commit(r_root)
3678
+ if first:
3679
+ con.execute("INSERT INTO meta VALUES (?, ?)", (f"first_commit:{r}", first))
3680
+ # Everything from here reads the store. Letting the indexer go first keeps its memory (most of a GB on a
3681
+ # large repo) from adding to what clustering and the pattern matchers hold.
3682
+ repo, timing, repos = ix.repo, ix.timing, dict(ix.repos)
3683
+ inc.release()
3684
+ del ix
3685
+ mark = time.perf_counter()
3686
+ multi = len(repos) > 1
3687
+ systems = {r: inc.systems(con, r) for r in repos}
3688
+ stats["systems"] = systems if multi else systems[repo]
3689
+ if inc.full:
3690
+ with con: # a workspace re-clusters every repo, so it rebuilds everything
3691
+ store.rebuild_derived(con, systems_of=None if multi else repo)
3692
+ timing["systems"] = round(time.perf_counter() - mark, 3)
3693
+ mark = time.perf_counter()
3694
+ stats["patterns"] = inc.patterns(con, repo)
3695
+ for other in list(repos)[1:]:
3696
+ with con:
3697
+ store.write_coverage(con, other, "patterns:structural", "0.1", "ok", None, stats["patterns"])
3698
+ tour = {r: tours.generate(con, r) for r in repos}
3699
+ stats["tour"] = tour if multi else tour[repo]
3700
+ timing["patterns_and_tour"] = round(time.perf_counter() - mark, 3)
3701
+ stats["stale_annotations"] = store.refresh_stale(con)
3702
+ if multi:
3703
+ stats["workspace"] = {"repos": {r: str(p) for r, p in repos.items()}, "cross_repo": _cross_counts(con)}
3704
+ files = sum(v.get("files", 0) for k, v in stats.items() if k.startswith(("tree-sitter", "generic")))
3705
+ lines = con.execute(f"SELECT COALESCE(SUM(span_end), 0) FROM nodes WHERE kind = 'file' AND repo_id IN ({','.join('?' * len(repos))})",
3706
+ list(repos)).fetchone()[0]
3707
+ total = round(time.perf_counter() - began, 3)
3708
+ stats["timing"] = {"total_seconds": total, **timing, "files": files, "lines": lines,
3709
+ "lines_per_second": round(lines / total) if total else 0}
3710
+ stats["incremental"] = inc.summary()
3711
+ with con: # kept in the store so a later reader can see what the map cost to build
3712
+ for r in repos:
3713
+ con.execute("INSERT OR REPLACE INTO meta VALUES (?, ?)", (f"timing:{r}", json.dumps(stats["timing"])))
3714
+ con.execute("INSERT OR REPLACE INTO extractor_coverage VALUES (?,?,?,?,?,?)",
3715
+ (r, "timing", "-", "ok", None, json.dumps(stats["timing"])))
3716
+ # Which Leyline made this map, noted once the whole of it is written: a newer one, or a run that stopped
3717
+ # part way (its systems, patterns or tour not yet redone), maps again even where no file changed
3718
+ # (see loop.refresh).
3719
+ con.execute("INSERT OR REPLACE INTO meta VALUES ('made_by', ?)", (incremental.code_version(),))
3720
+ inc.finish(con)
3721
+ if _verify and stats["incremental"]["mode"] == "incremental" and os.environ.get("LEYLINE_VERIFY"):
3722
+ stats["incremental"]["differs"] = incremental.verify(db_path, members, exact, scip, listed)
3723
+ inc = None
3724
+ return stats
3725
+ finally:
3726
+ if inc is not None:
3727
+ inc.abandon()
3728
+ con.close()
3729
+
3730
+
3731
+ def _cross_counts(con) -> list[dict]:
3732
+ from .query import cross_repo
3733
+ return cross_repo(con)["pairs"]