local-codebase-indexer 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- local_codebase_indexer/__init__.py +3 -0
- local_codebase_indexer/__main__.py +5 -0
- local_codebase_indexer/callgraph.py +279 -0
- local_codebase_indexer/cli.py +122 -0
- local_codebase_indexer/core.py +323 -0
- local_codebase_indexer/cscalls.py +304 -0
- local_codebase_indexer/csdata.py +450 -0
- local_codebase_indexer/dataflow.py +250 -0
- local_codebase_indexer/db.py +194 -0
- local_codebase_indexer/embedder.py +137 -0
- local_codebase_indexer/langparse.py +569 -0
- local_codebase_indexer/mcp_server.py +136 -0
- local_codebase_indexer/query.py +756 -0
- local_codebase_indexer/scanner.py +101 -0
- local_codebase_indexer/sqlaccess.py +301 -0
- local_codebase_indexer/synonyms.py +101 -0
- local_codebase_indexer-0.2.0.dist-info/METADATA +118 -0
- local_codebase_indexer-0.2.0.dist-info/RECORD +21 -0
- local_codebase_indexer-0.2.0.dist-info/WHEEL +5 -0
- local_codebase_indexer-0.2.0.dist-info/entry_points.txt +2 -0
- local_codebase_indexer-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
"""Incremental indexing pipeline (index.md section 3, step 1)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import os
|
|
7
|
+
import posixpath
|
|
8
|
+
import re
|
|
9
|
+
import sqlite3
|
|
10
|
+
import subprocess
|
|
11
|
+
import time
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
from . import db as dbmod
|
|
16
|
+
from .callgraph import resolve_calls
|
|
17
|
+
from .embedder import get_embedder, split_identifier
|
|
18
|
+
from .langparse import CLASS_LIKE, parse_file
|
|
19
|
+
from .scanner import iter_files
|
|
20
|
+
|
|
21
|
+
_TYPE_KINDS = tuple(CLASS_LIKE)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def decode_bytes(data: bytes) -> str:
|
|
25
|
+
for enc in ("utf-8-sig", "big5", "cp1252"):
|
|
26
|
+
try:
|
|
27
|
+
return data.decode(enc)
|
|
28
|
+
except UnicodeDecodeError:
|
|
29
|
+
continue
|
|
30
|
+
return data.decode("utf-8", errors="replace")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
_CJK_RUN = re.compile(r"[\u4e00-\u9fff]+")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def cjk_bigrams(run: str) -> list[str]:
|
|
37
|
+
"""Overlapping character bigrams of a Chinese run (the run itself when it is a single character)."""
|
|
38
|
+
return [run[i:i + 2] for i in range(len(run) - 1)] or [run]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def search_text_for(header: str, body: str) -> str:
|
|
42
|
+
"""Text for FTS5: original tokens plus camelCase / snake_case sub-words."""
|
|
43
|
+
text = header + "\n" + body
|
|
44
|
+
extra = []
|
|
45
|
+
for ident in set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text)):
|
|
46
|
+
parts = split_identifier(ident)
|
|
47
|
+
if len(parts) > 1:
|
|
48
|
+
extra.extend(parts)
|
|
49
|
+
# FTS5's unicode61 tokenizer keeps a whole run of Chinese characters as one token,
|
|
50
|
+
# so index overlapping character bigrams as separate words
|
|
51
|
+
for run in set(_CJK_RUN.findall(text)):
|
|
52
|
+
extra.extend(cjk_bigrams(run))
|
|
53
|
+
return text + "\n" + " ".join(extra)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def embedding_text_for(header: str, body: str) -> str:
|
|
57
|
+
return (header + "\n" + body)[:2500]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def git_info(root: str) -> dict | None:
|
|
61
|
+
"""HEAD commit, branch and number of uncommitted paths; None outside a git work tree."""
|
|
62
|
+
def run(*args: str) -> str | None:
|
|
63
|
+
try:
|
|
64
|
+
r = subprocess.run(["git", "-C", root, *args], capture_output=True, text=True, timeout=15)
|
|
65
|
+
except (OSError, subprocess.SubprocessError):
|
|
66
|
+
return None
|
|
67
|
+
return r.stdout if r.returncode == 0 else None
|
|
68
|
+
|
|
69
|
+
head = run("rev-parse", "HEAD")
|
|
70
|
+
if head is None:
|
|
71
|
+
return None
|
|
72
|
+
branch = (run("rev-parse", "--abbrev-ref", "HEAD") or "").strip()
|
|
73
|
+
porcelain = run("status", "--porcelain") or ""
|
|
74
|
+
return {
|
|
75
|
+
"commit": head.strip(),
|
|
76
|
+
"branch": branch,
|
|
77
|
+
"dirty_files": len([ln for ln in porcelain.splitlines() if ln.strip()]),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class IndexStats:
|
|
83
|
+
scanned: int = 0
|
|
84
|
+
added: int = 0
|
|
85
|
+
updated: int = 0
|
|
86
|
+
unchanged: int = 0
|
|
87
|
+
deleted: int = 0
|
|
88
|
+
skipped: int = 0
|
|
89
|
+
failed: int = 0
|
|
90
|
+
chunks: int = 0
|
|
91
|
+
symbols: int = 0
|
|
92
|
+
warnings: list[str] = field(default_factory=list)
|
|
93
|
+
|
|
94
|
+
def as_dict(self) -> dict:
|
|
95
|
+
return self.__dict__.copy()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Indexer:
|
|
99
|
+
def __init__(self, root: str, db_path: str | None = None, embedder=None):
|
|
100
|
+
self.root = os.path.abspath(root)
|
|
101
|
+
self.db_path = db_path or dbmod.default_db_path(self.root)
|
|
102
|
+
self.conn = dbmod.connect(self.db_path)
|
|
103
|
+
self.embedder = embedder or get_embedder()
|
|
104
|
+
|
|
105
|
+
def close(self) -> None:
|
|
106
|
+
self.conn.close()
|
|
107
|
+
|
|
108
|
+
# ------------------------------------------------------------------ public
|
|
109
|
+
def update(self, force: bool = False) -> IndexStats:
|
|
110
|
+
stats = IndexStats()
|
|
111
|
+
conn = self.conn
|
|
112
|
+
|
|
113
|
+
if (dbmod.get_meta(conn, "embedder") != self.embedder.name
|
|
114
|
+
or dbmod.get_meta(conn, "index_version") != dbmod.INDEX_VERSION):
|
|
115
|
+
# different embedding space or data layout (or first run): rebuild everything
|
|
116
|
+
dbmod.clear_all(conn)
|
|
117
|
+
dbmod.set_meta(conn, "embedder", self.embedder.name)
|
|
118
|
+
dbmod.set_meta(conn, "index_version", dbmod.INDEX_VERSION)
|
|
119
|
+
conn.commit()
|
|
120
|
+
|
|
121
|
+
existing = {r["file_path"]: (r["last_modified"], r["md5_hash"]) for r in
|
|
122
|
+
conn.execute("SELECT file_path, last_modified, md5_hash FROM files")}
|
|
123
|
+
seen: set[str] = set()
|
|
124
|
+
changed = False
|
|
125
|
+
|
|
126
|
+
for sf in iter_files(self.root):
|
|
127
|
+
stats.scanned += 1
|
|
128
|
+
seen.add(sf.rel_path)
|
|
129
|
+
prev = existing.get(sf.rel_path)
|
|
130
|
+
if prev and not force and prev[0] == sf.mtime_ns:
|
|
131
|
+
stats.unchanged += 1
|
|
132
|
+
continue
|
|
133
|
+
try:
|
|
134
|
+
with open(sf.abs_path, "rb") as f:
|
|
135
|
+
data = f.read()
|
|
136
|
+
except OSError as exc:
|
|
137
|
+
self._record_failure(sf.rel_path, f"{type(exc).__name__}: {exc}")
|
|
138
|
+
stats.failed += 1
|
|
139
|
+
changed = True
|
|
140
|
+
stats.warnings.append(f"{sf.rel_path}: {exc}")
|
|
141
|
+
continue
|
|
142
|
+
if b"\x00" in data[:8192]:
|
|
143
|
+
stats.skipped += 1
|
|
144
|
+
continue
|
|
145
|
+
md5 = hashlib.md5(data).hexdigest()
|
|
146
|
+
if prev and not force and prev[1] == md5:
|
|
147
|
+
conn.execute("UPDATE files SET last_modified=? WHERE file_path=?", (sf.mtime_ns, sf.rel_path))
|
|
148
|
+
stats.unchanged += 1
|
|
149
|
+
continue
|
|
150
|
+
try:
|
|
151
|
+
self._index_file(sf.rel_path, sf.language, sf.mtime_ns, md5, decode_bytes(data))
|
|
152
|
+
except Exception as exc: # one bad file must not abort the run
|
|
153
|
+
self._record_failure(sf.rel_path, f"{type(exc).__name__}: {exc}")
|
|
154
|
+
stats.failed += 1
|
|
155
|
+
changed = True
|
|
156
|
+
stats.warnings.append(f"{sf.rel_path}: {type(exc).__name__}: {exc}")
|
|
157
|
+
continue
|
|
158
|
+
changed = True
|
|
159
|
+
if prev:
|
|
160
|
+
stats.updated += 1
|
|
161
|
+
else:
|
|
162
|
+
stats.added += 1
|
|
163
|
+
|
|
164
|
+
for gone in set(existing) - seen:
|
|
165
|
+
self._remove_file(gone)
|
|
166
|
+
stats.deleted += 1
|
|
167
|
+
changed = True
|
|
168
|
+
|
|
169
|
+
# failures for files that are no longer scanned (deleted, newly ignored) are obsolete
|
|
170
|
+
conn.executemany("DELETE FROM index_failures WHERE file_path=?",
|
|
171
|
+
[(r[0],) for r in conn.execute("SELECT file_path FROM index_failures").fetchall()
|
|
172
|
+
if r[0] not in seen])
|
|
173
|
+
if changed or force:
|
|
174
|
+
resolve_dependencies(conn)
|
|
175
|
+
resolve_calls(conn)
|
|
176
|
+
self._record_run_meta()
|
|
177
|
+
conn.commit()
|
|
178
|
+
stats.chunks = conn.execute("SELECT COUNT(*) FROM chunk_metadata").fetchone()[0]
|
|
179
|
+
stats.symbols = conn.execute("SELECT COUNT(*) FROM symbols").fetchone()[0]
|
|
180
|
+
return stats
|
|
181
|
+
|
|
182
|
+
# ------------------------------------------------------------------ internals
|
|
183
|
+
def _record_run_meta(self) -> None:
|
|
184
|
+
conn = self.conn
|
|
185
|
+
dbmod.set_meta(conn, "last_indexed_at", str(int(time.time())))
|
|
186
|
+
info = git_info(self.root)
|
|
187
|
+
if info is None:
|
|
188
|
+
for k in ("git_commit", "git_branch", "git_dirty_files"):
|
|
189
|
+
conn.execute("DELETE FROM meta WHERE key=?", (k,))
|
|
190
|
+
else:
|
|
191
|
+
dbmod.set_meta(conn, "git_commit", info["commit"])
|
|
192
|
+
dbmod.set_meta(conn, "git_branch", info["branch"])
|
|
193
|
+
dbmod.set_meta(conn, "git_dirty_files", str(info["dirty_files"]))
|
|
194
|
+
|
|
195
|
+
def _record_failure(self, rel_path: str, error: str) -> None:
|
|
196
|
+
"""A file that cannot be indexed must not leave its previous (now stale) data behind."""
|
|
197
|
+
with self.conn:
|
|
198
|
+
self._remove_file(rel_path)
|
|
199
|
+
self.conn.execute(
|
|
200
|
+
"INSERT OR REPLACE INTO index_failures(file_path, error, failed_at) VALUES(?,?,?)",
|
|
201
|
+
(rel_path, error[:500], int(time.time())),
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def _remove_file(self, rel_path: str) -> None:
|
|
205
|
+
c = self.conn
|
|
206
|
+
ids = [r[0] for r in c.execute("SELECT rowid FROM chunk_metadata WHERE file_path=?", (rel_path,))]
|
|
207
|
+
c.executemany("DELETE FROM vss_chunks WHERE rowid=?", [(i,) for i in ids])
|
|
208
|
+
c.executemany("DELETE FROM chunks_fts WHERE rowid=?", [(i,) for i in ids])
|
|
209
|
+
c.execute("DELETE FROM chunk_metadata WHERE file_path=?", (rel_path,))
|
|
210
|
+
c.execute("DELETE FROM symbols WHERE file_path=?", (rel_path,))
|
|
211
|
+
c.execute("DELETE FROM raw_refs WHERE file_path=?", (rel_path,))
|
|
212
|
+
c.execute("DELETE FROM data_access WHERE file_path=?", (rel_path,))
|
|
213
|
+
c.execute("DELETE FROM call_sites WHERE file_path=?", (rel_path,))
|
|
214
|
+
c.execute("DELETE FROM files WHERE file_path=?", (rel_path,))
|
|
215
|
+
c.execute("DELETE FROM index_failures WHERE file_path=?", (rel_path,))
|
|
216
|
+
|
|
217
|
+
def _index_file(self, rel_path: str, language: str, mtime_ns: int, md5: str, text: str) -> None:
|
|
218
|
+
parsed = parse_file(language, text, rel_path)
|
|
219
|
+
embeddings = self.embedder.embed(
|
|
220
|
+
[embedding_text_for(ch.header, ch.text) for ch in parsed.chunks]
|
|
221
|
+
)
|
|
222
|
+
c = self.conn
|
|
223
|
+
with c:
|
|
224
|
+
self._remove_file(rel_path)
|
|
225
|
+
c.execute(
|
|
226
|
+
"INSERT INTO files(file_path, last_modified, md5_hash, language) VALUES(?,?,?,?)",
|
|
227
|
+
(rel_path, mtime_ns, md5, language),
|
|
228
|
+
)
|
|
229
|
+
c.executemany(
|
|
230
|
+
"INSERT OR REPLACE INTO symbols(id, file_path, symbol_name, symbol_type, parent_id,"
|
|
231
|
+
" start_line, end_line, signature, qualified_name, type_name, min_args, max_args)"
|
|
232
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
233
|
+
[(s.id, rel_path, s.name, s.type, s.parent_id, s.start, s.end, s.signature, s.qname,
|
|
234
|
+
s.type_name or None, s.min_args, s.max_args)
|
|
235
|
+
for s in parsed.symbols],
|
|
236
|
+
)
|
|
237
|
+
c.executemany(
|
|
238
|
+
"INSERT INTO data_access(file_path, source_id, line, target_key, target_name, qualifier,"
|
|
239
|
+
" target_kind, access, op, via, confidence, evidence, detail) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
240
|
+
[(rel_path, a.source_id, a.line, a.key, a.name, a.qualifier, a.kind, a.access, a.op, a.via,
|
|
241
|
+
a.confidence, a.evidence, a.detail) for a in parsed.accesses],
|
|
242
|
+
)
|
|
243
|
+
c.executemany(
|
|
244
|
+
"INSERT INTO call_sites(file_path, caller_id, line, name, argc, recv_kind, recv_name, recv_type,"
|
|
245
|
+
" recv_expr, is_ctor) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
|
246
|
+
[(rel_path, cs.caller_id, cs.line, cs.name, cs.argc, cs.recv_kind, cs.recv_name, cs.recv_type,
|
|
247
|
+
cs.recv_expr, int(cs.is_ctor)) for cs in parsed.call_sites],
|
|
248
|
+
)
|
|
249
|
+
c.executemany(
|
|
250
|
+
"INSERT OR IGNORE INTO raw_refs(file_path, kind, name, line, owner) VALUES(?,?,?,?,?)",
|
|
251
|
+
[(rel_path, r.kind, r.name, r.line, r.owner) for r in parsed.refs],
|
|
252
|
+
)
|
|
253
|
+
for ch, emb in zip(parsed.chunks, embeddings):
|
|
254
|
+
cur = c.execute(
|
|
255
|
+
"INSERT INTO chunk_metadata(file_path, start_line, end_line, raw_text, symbol_id)"
|
|
256
|
+
" VALUES(?,?,?,?,?)",
|
|
257
|
+
(rel_path, ch.start, ch.end, ch.text, ch.symbol_id),
|
|
258
|
+
)
|
|
259
|
+
rid = cur.lastrowid
|
|
260
|
+
c.execute("INSERT INTO vss_chunks(rowid, chunk_embedding) VALUES(?,?)", (rid, emb.tobytes()))
|
|
261
|
+
c.execute("INSERT INTO chunks_fts(rowid, search_text) VALUES(?,?)",
|
|
262
|
+
(rid, search_text_for(ch.header, ch.text)))
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ---------------------------------------------------------------------- dependency resolution
|
|
266
|
+
|
|
267
|
+
def _resolve_relative(source: str, spec: str, files: set[str]) -> str | None:
|
|
268
|
+
base = posixpath.normpath(posixpath.join(posixpath.dirname(source), spec))
|
|
269
|
+
for cand in (base, *(base + e for e in (".ts", ".tsx", ".js", ".jsx", ".mjs", ".py", ".cs")),
|
|
270
|
+
*(posixpath.join(base, "index" + e) for e in (".ts", ".tsx", ".js", ".jsx"))):
|
|
271
|
+
if cand in files:
|
|
272
|
+
return cand
|
|
273
|
+
return None
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def resolve_dependencies(conn: sqlite3.Connection) -> None:
|
|
277
|
+
"""Rebuild the file-level dependency graph from raw_refs and symbols."""
|
|
278
|
+
conn.execute("DELETE FROM dependencies")
|
|
279
|
+
files = {r[0] for r in conn.execute("SELECT file_path FROM files")}
|
|
280
|
+
|
|
281
|
+
type_files: dict[str, set[str]] = defaultdict(set)
|
|
282
|
+
for r in conn.execute(
|
|
283
|
+
f"SELECT symbol_name, file_path FROM symbols WHERE symbol_type IN ({','.join('?' * len(_TYPE_KINDS))})",
|
|
284
|
+
_TYPE_KINDS,
|
|
285
|
+
):
|
|
286
|
+
type_files[r[0]].add(r[1])
|
|
287
|
+
|
|
288
|
+
ns_files: dict[str, set[str]] = defaultdict(set)
|
|
289
|
+
for r in conn.execute("SELECT symbol_name, file_path FROM symbols WHERE symbol_type='namespace'"):
|
|
290
|
+
ns_files[r[0]].add(r[1])
|
|
291
|
+
|
|
292
|
+
deps: set[tuple[str, str, str]] = set()
|
|
293
|
+
|
|
294
|
+
for r in conn.execute("SELECT file_path, name FROM raw_refs WHERE kind='import'"):
|
|
295
|
+
src, name = r[0], r[1]
|
|
296
|
+
targets: set[str] = set()
|
|
297
|
+
if name.startswith("."):
|
|
298
|
+
t = _resolve_relative(src, name, files)
|
|
299
|
+
if t:
|
|
300
|
+
targets.add(t)
|
|
301
|
+
elif name in ns_files:
|
|
302
|
+
targets |= ns_files[name]
|
|
303
|
+
else:
|
|
304
|
+
slug = name.replace(".", "/")
|
|
305
|
+
for cand in (slug + ".py", slug + "/__init__.py"):
|
|
306
|
+
targets |= {f for f in files if f == cand or f.endswith("/" + cand)}
|
|
307
|
+
for t in targets:
|
|
308
|
+
if t != src:
|
|
309
|
+
deps.add((src, t, "import"))
|
|
310
|
+
|
|
311
|
+
for r in conn.execute("SELECT file_path, name FROM raw_refs WHERE kind='base'"):
|
|
312
|
+
for t in type_files.get(r[1], ()):
|
|
313
|
+
if t != r[0]:
|
|
314
|
+
deps.add((r[0], t, "inheritance"))
|
|
315
|
+
|
|
316
|
+
for r in conn.execute("SELECT DISTINCT file_path, name FROM raw_refs WHERE kind='ident'"):
|
|
317
|
+
targets = type_files.get(r[1])
|
|
318
|
+
if targets and len(targets) <= 5:
|
|
319
|
+
for t in targets:
|
|
320
|
+
if t != r[0]:
|
|
321
|
+
deps.add((r[0], t, "invocation"))
|
|
322
|
+
|
|
323
|
+
conn.executemany("INSERT OR IGNORE INTO dependencies VALUES(?,?,?)", sorted(deps))
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"""C# call sites with a light local type environment, for symbol-level call resolution.
|
|
2
|
+
|
|
3
|
+
Only facts that the syntax tree states are recorded here (receiver kind and name, declared types of
|
|
4
|
+
parameters and locals, ``var x = new T()`` / casts / ``as``). Matching a call to a method definition happens
|
|
5
|
+
later in ``callgraph.py``, once every file is indexed.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
_SCOPES = {
|
|
14
|
+
"method_declaration", "constructor_declaration", "destructor_declaration", "operator_declaration",
|
|
15
|
+
"indexer_declaration", "property_declaration",
|
|
16
|
+
}
|
|
17
|
+
_TASK = re.compile(r"^(?:[\w.]+\.)?(?:Task|ValueTask)\s*<(.+)>$")
|
|
18
|
+
_OUTER = re.compile(r"^([\w.]+)\s*(?:<.*>)?$", re.S)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class CallSite:
|
|
23
|
+
caller_id: str | None
|
|
24
|
+
line: int
|
|
25
|
+
name: str
|
|
26
|
+
argc: int
|
|
27
|
+
recv_kind: str # none | this | base | name | member | new | cast | expr | type
|
|
28
|
+
recv_name: str = ""
|
|
29
|
+
recv_type: str = "" # type known from the syntax (declared type, new T(), cast); "" when unknown
|
|
30
|
+
recv_expr: str = "" # "kind|name|method|argc|rtype" of the call that produced the receiver
|
|
31
|
+
is_ctor: bool = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def norm_type(text: str) -> str:
|
|
35
|
+
"""Outer type name without namespace, generic arguments, nullability or array brackets.
|
|
36
|
+
``Task<T>`` / ``ValueTask<T>`` become ``T`` (the type a caller works with after ``await``)."""
|
|
37
|
+
t = (text or "").strip()
|
|
38
|
+
t = re.sub(r"^(?:ref|out|in|readonly|const|params)\s+", "", t)
|
|
39
|
+
t = t.replace("global::", "")
|
|
40
|
+
for _ in range(3):
|
|
41
|
+
m = _TASK.match(t)
|
|
42
|
+
if not m:
|
|
43
|
+
break
|
|
44
|
+
t = m.group(1).strip()
|
|
45
|
+
t = re.sub(r"(?:\[[,\s]*\]|\?)+$", "", t).strip()
|
|
46
|
+
if t.startswith("(") or t in ("var", "dynamic", "object", ""):
|
|
47
|
+
return ""
|
|
48
|
+
m = _OUTER.match(t)
|
|
49
|
+
return m.group(1).split(".")[-1] if m else ""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _text(n, src: bytes) -> str:
|
|
53
|
+
return src[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def member_info(node, src: bytes) -> tuple[str, int, int]:
|
|
57
|
+
"""(type name, min args, max args) for a C# member node. Type is the return / declared type; the argument
|
|
58
|
+
counts are -1 for members without a parameter list."""
|
|
59
|
+
t = node.type
|
|
60
|
+
type_name = ""
|
|
61
|
+
if t == "method_declaration":
|
|
62
|
+
r = node.child_by_field_name("returns")
|
|
63
|
+
type_name = norm_type(_text(r, src)) if r is not None else ""
|
|
64
|
+
elif t in ("property_declaration", "indexer_declaration"):
|
|
65
|
+
r = node.child_by_field_name("type")
|
|
66
|
+
type_name = norm_type(_text(r, src)) if r is not None else ""
|
|
67
|
+
elif t in ("field_declaration", "event_field_declaration"):
|
|
68
|
+
for c in node.children:
|
|
69
|
+
if c.type == "variable_declaration":
|
|
70
|
+
r = c.child_by_field_name("type")
|
|
71
|
+
type_name = norm_type(_text(r, src)) if r is not None else ""
|
|
72
|
+
plist = node.child_by_field_name("parameters")
|
|
73
|
+
if plist is None or t not in ("method_declaration", "constructor_declaration", "operator_declaration"):
|
|
74
|
+
return type_name, -1, -1
|
|
75
|
+
fixed = optional = 0
|
|
76
|
+
variadic = any(c.type == "params" or _text(c, src) == "params" for c in plist.children)
|
|
77
|
+
for p in plist.named_children:
|
|
78
|
+
if p.type != "parameter":
|
|
79
|
+
continue
|
|
80
|
+
tn, nm = p.child_by_field_name("type"), p.child_by_field_name("name")
|
|
81
|
+
declared = {(x.start_byte, x.end_byte) for x in (tn, nm) if x is not None}
|
|
82
|
+
has_default = any(c.type == "=" for c in p.children) or any(
|
|
83
|
+
c.is_named and c.type not in ("modifier", "attribute_list") and (c.start_byte, c.end_byte) not in declared
|
|
84
|
+
for c in p.children)
|
|
85
|
+
fixed += 1
|
|
86
|
+
optional += 1 if has_default else 0
|
|
87
|
+
return type_name, fixed - optional, (99 if variadic else fixed)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class _Calls:
|
|
91
|
+
def __init__(self, src: bytes, symbols: list):
|
|
92
|
+
self.src = src
|
|
93
|
+
spans = [(s.end - s.start, s) for s in symbols if s.type != "namespace"]
|
|
94
|
+
spans.sort(key=lambda t: (t[0], t[1].start))
|
|
95
|
+
self.spans = [s for _, s in spans]
|
|
96
|
+
self.out: list[CallSite] = []
|
|
97
|
+
|
|
98
|
+
def text(self, n) -> str:
|
|
99
|
+
return _text(n, self.src)
|
|
100
|
+
|
|
101
|
+
def locate(self, line: int):
|
|
102
|
+
for s in self.spans:
|
|
103
|
+
if s.start <= line <= s.end:
|
|
104
|
+
return s
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
# ---------------------------------------------------------------- local environment
|
|
108
|
+
def _set(self, env: dict, name: str, typ: str) -> None:
|
|
109
|
+
if name in env and env[name] != typ:
|
|
110
|
+
env[name] = "" # declared twice with different types: unknown
|
|
111
|
+
else:
|
|
112
|
+
env[name] = typ
|
|
113
|
+
|
|
114
|
+
def _unwrap(self, n):
|
|
115
|
+
while n is not None and n.type in ("parenthesized_expression", "await_expression"):
|
|
116
|
+
kids = list(n.named_children)
|
|
117
|
+
n = kids[-1] if kids else None
|
|
118
|
+
return n
|
|
119
|
+
|
|
120
|
+
def _infer_initializer(self, init, env) -> str:
|
|
121
|
+
n = self._unwrap(init)
|
|
122
|
+
if n is None:
|
|
123
|
+
return ""
|
|
124
|
+
if n.type == "object_creation_expression":
|
|
125
|
+
t = n.child_by_field_name("type")
|
|
126
|
+
return norm_type(self.text(t)) if t is not None else ""
|
|
127
|
+
if n.type == "cast_expression":
|
|
128
|
+
kids = list(n.named_children)
|
|
129
|
+
return norm_type(self.text(kids[0])) if kids else ""
|
|
130
|
+
if n.type == "as_expression":
|
|
131
|
+
kids = list(n.named_children)
|
|
132
|
+
return norm_type(self.text(kids[-1])) if len(kids) > 1 else ""
|
|
133
|
+
if n.type == "invocation_expression":
|
|
134
|
+
info = self._call_info(n, env, depth=1)
|
|
135
|
+
if info is not None:
|
|
136
|
+
kind, name, recv_type, _expr, method, argc = info
|
|
137
|
+
if kind in ("none", "this", "name", "member"):
|
|
138
|
+
return f"@call:{kind}|{name}|{method}|{argc}|{recv_type}"
|
|
139
|
+
return ""
|
|
140
|
+
|
|
141
|
+
def collect_locals(self, scope) -> dict:
|
|
142
|
+
env: dict[str, str] = {}
|
|
143
|
+
plist = scope.child_by_field_name("parameters")
|
|
144
|
+
if plist is not None:
|
|
145
|
+
for p in plist.named_children:
|
|
146
|
+
if p.type == "parameter":
|
|
147
|
+
t, nm = p.child_by_field_name("type"), p.child_by_field_name("name")
|
|
148
|
+
if t is not None and nm is not None:
|
|
149
|
+
self._set(env, self.text(nm), norm_type(self.text(t)))
|
|
150
|
+
stack = list(scope.children)
|
|
151
|
+
while stack:
|
|
152
|
+
n = stack.pop()
|
|
153
|
+
ty = n.type
|
|
154
|
+
if ty == "variable_declaration":
|
|
155
|
+
tnode = n.child_by_field_name("type")
|
|
156
|
+
implicit = tnode is None or tnode.type == "implicit_type"
|
|
157
|
+
declared = "" if implicit else norm_type(self.text(tnode))
|
|
158
|
+
for d in n.named_children:
|
|
159
|
+
if d.type != "variable_declarator":
|
|
160
|
+
continue
|
|
161
|
+
ident = next((c for c in d.named_children if c.type == "identifier"), None)
|
|
162
|
+
if ident is None:
|
|
163
|
+
continue
|
|
164
|
+
init = None
|
|
165
|
+
for c in d.named_children:
|
|
166
|
+
if c.start_byte != ident.start_byte:
|
|
167
|
+
init = c
|
|
168
|
+
typ = declared if not implicit else (self._infer_initializer(init, env) if init is not None else "")
|
|
169
|
+
self._set(env, self.text(ident), typ)
|
|
170
|
+
elif ty == "foreach_statement":
|
|
171
|
+
tnode, left = n.child_by_field_name("type"), n.child_by_field_name("left")
|
|
172
|
+
if left is not None and left.type == "identifier":
|
|
173
|
+
typ = "" if (tnode is None or tnode.type == "implicit_type") else norm_type(self.text(tnode))
|
|
174
|
+
self._set(env, self.text(left), typ)
|
|
175
|
+
elif ty == "catch_declaration":
|
|
176
|
+
kids = list(n.named_children)
|
|
177
|
+
if len(kids) >= 2 and kids[-1].type == "identifier":
|
|
178
|
+
self._set(env, self.text(kids[-1]), norm_type(self.text(kids[0])))
|
|
179
|
+
elif ty == "declaration_pattern":
|
|
180
|
+
kids = list(n.named_children)
|
|
181
|
+
if len(kids) >= 2:
|
|
182
|
+
self._set(env, self.text(kids[-1]), norm_type(self.text(kids[0])))
|
|
183
|
+
stack.extend(n.children)
|
|
184
|
+
return env
|
|
185
|
+
|
|
186
|
+
# ---------------------------------------------------------------- call classification
|
|
187
|
+
@staticmethod
|
|
188
|
+
def _method_name(func, text) -> str | None:
|
|
189
|
+
if func is None:
|
|
190
|
+
return None
|
|
191
|
+
t = func.type
|
|
192
|
+
if t == "identifier":
|
|
193
|
+
return text(func)
|
|
194
|
+
if t == "generic_name":
|
|
195
|
+
for c in func.children:
|
|
196
|
+
if c.type == "identifier":
|
|
197
|
+
return text(c)
|
|
198
|
+
if t == "member_access_expression":
|
|
199
|
+
return _Calls._method_name(func.child_by_field_name("name"), text)
|
|
200
|
+
return None
|
|
201
|
+
|
|
202
|
+
def _classify_receiver(self, expr, env, depth: int) -> tuple[str, str, str, str]:
|
|
203
|
+
"""(kind, name, type, expr) of a receiver expression."""
|
|
204
|
+
n = self._unwrap(expr)
|
|
205
|
+
if n is None:
|
|
206
|
+
return "expr", "", "", ""
|
|
207
|
+
t = n.type
|
|
208
|
+
if t == "identifier":
|
|
209
|
+
nm = self.text(n)
|
|
210
|
+
typ = env.get(nm, "")
|
|
211
|
+
if typ.startswith("@call:"):
|
|
212
|
+
return "name", nm, "", typ[len("@call:"):]
|
|
213
|
+
return "name", nm, typ, ""
|
|
214
|
+
if t == "this_expression":
|
|
215
|
+
return "this", "", "", ""
|
|
216
|
+
if t == "base_expression":
|
|
217
|
+
return "base", "", "", ""
|
|
218
|
+
if t == "member_access_expression":
|
|
219
|
+
inner = n.child_by_field_name("expression")
|
|
220
|
+
nm = n.child_by_field_name("name")
|
|
221
|
+
if inner is not None and inner.type == "this_expression" and nm is not None:
|
|
222
|
+
return "member", self.text(nm), "", ""
|
|
223
|
+
return "expr", "", "", ""
|
|
224
|
+
if t == "object_creation_expression":
|
|
225
|
+
tn = n.child_by_field_name("type")
|
|
226
|
+
return "new", "", norm_type(self.text(tn)) if tn is not None else "", ""
|
|
227
|
+
if t == "cast_expression":
|
|
228
|
+
kids = list(n.named_children)
|
|
229
|
+
return "cast", "", norm_type(self.text(kids[0])) if kids else "", ""
|
|
230
|
+
if t == "as_expression":
|
|
231
|
+
kids = list(n.named_children)
|
|
232
|
+
return "cast", "", norm_type(self.text(kids[-1])) if len(kids) > 1 else "", ""
|
|
233
|
+
if t == "invocation_expression" and depth < 2:
|
|
234
|
+
info = self._call_info(n, env, depth + 1)
|
|
235
|
+
if info is not None:
|
|
236
|
+
kind, name, recv_type, _expr, method, argc = info
|
|
237
|
+
if kind in ("none", "this", "name", "member"):
|
|
238
|
+
return "expr", "", "", f"{kind}|{name}|{method}|{argc}|{recv_type}"
|
|
239
|
+
return "expr", "", "", ""
|
|
240
|
+
|
|
241
|
+
def _call_info(self, n, env, depth: int):
|
|
242
|
+
"""(recv_kind, recv_name, recv_type, recv_expr, method name, argc) of an invocation node."""
|
|
243
|
+
func = n.child_by_field_name("function")
|
|
244
|
+
if func is None:
|
|
245
|
+
return None
|
|
246
|
+
args = n.child_by_field_name("arguments")
|
|
247
|
+
argc = sum(1 for a in args.named_children if a.type == "argument") if args is not None else 0
|
|
248
|
+
name = self._method_name(func, self.text)
|
|
249
|
+
if func.type in ("identifier", "generic_name"):
|
|
250
|
+
return ("none", "", "", "", name, argc) if name else None
|
|
251
|
+
if func.type == "member_access_expression":
|
|
252
|
+
if not name:
|
|
253
|
+
return None
|
|
254
|
+
kind, rname, rtype, rexpr = self._classify_receiver(func.child_by_field_name("expression"), env, depth)
|
|
255
|
+
return kind, rname, rtype, rexpr, name, argc
|
|
256
|
+
if func.type == "conditional_access_expression":
|
|
257
|
+
binding = next((c for c in func.children if c.type == "member_binding_expression"), None)
|
|
258
|
+
if binding is None:
|
|
259
|
+
return None
|
|
260
|
+
ident = next((c for c in binding.children if c.type in ("identifier", "generic_name")), None)
|
|
261
|
+
name = self._method_name(ident, self.text)
|
|
262
|
+
cond = func.child_by_field_name("condition") or (func.named_children[0] if func.named_children else None)
|
|
263
|
+
if not name or cond is None:
|
|
264
|
+
return None
|
|
265
|
+
kind, rname, rtype, rexpr = self._classify_receiver(cond, env, depth)
|
|
266
|
+
return kind, rname, rtype, rexpr, name, argc
|
|
267
|
+
return None
|
|
268
|
+
|
|
269
|
+
# ---------------------------------------------------------------- traversal
|
|
270
|
+
def run(self, root) -> list[CallSite]:
|
|
271
|
+
stack: list = [("node", root, {})]
|
|
272
|
+
while stack:
|
|
273
|
+
_kind, n, env = stack.pop()
|
|
274
|
+
if n.type in _SCOPES:
|
|
275
|
+
env = self.collect_locals(n)
|
|
276
|
+
elif n.type == "field_declaration":
|
|
277
|
+
env = {}
|
|
278
|
+
if n.type == "invocation_expression":
|
|
279
|
+
info = self._call_info(n, env, 0)
|
|
280
|
+
if info is not None:
|
|
281
|
+
kind, rname, rtype, rexpr, method, argc = info
|
|
282
|
+
self._add(n, method, argc, kind, rname, rtype, rexpr, False)
|
|
283
|
+
elif n.type == "object_creation_expression":
|
|
284
|
+
tn = n.child_by_field_name("type")
|
|
285
|
+
tname = norm_type(self.text(tn)) if tn is not None else ""
|
|
286
|
+
if tname:
|
|
287
|
+
args = n.child_by_field_name("arguments")
|
|
288
|
+
argc = sum(1 for a in args.named_children if a.type == "argument") if args is not None else 0
|
|
289
|
+
self._add(n, tname, argc, "type", tname, "", "", True)
|
|
290
|
+
for c in reversed(n.children):
|
|
291
|
+
stack.append(("node", c, env))
|
|
292
|
+
return self.out
|
|
293
|
+
|
|
294
|
+
def _add(self, n, name, argc, kind, rname, rtype, rexpr, is_ctor) -> None:
|
|
295
|
+
line = n.start_point[0] + 1
|
|
296
|
+
sym = self.locate(line)
|
|
297
|
+
self.out.append(CallSite(sym.id if sym is not None else None, line, name, argc, kind, rname, rtype, rexpr, is_ctor))
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def extract_call_sites(root, src: bytes, symbols: list) -> list[CallSite]:
|
|
301
|
+
try:
|
|
302
|
+
return _Calls(src, symbols).run(root)
|
|
303
|
+
except RecursionError:
|
|
304
|
+
return []
|